From 88ebf599e88a865af855243efe36ea962054be38 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Thu, 11 Jul 2024 12:18:09 +0800 Subject: [PATCH 01/88] add file-to-blackhole ilogtail benchmark --- .../LICENSE_OF_TESTENGINE_DEPENDENCIES.md | 3 +- test/benchmark/benchmark_docker_test.go | 68 ++++++++++ test/benchmark/main_test.go | 123 ++++++++++++++++++ .../a.log | 0 .../case.feature | 39 ++++++ .../docker-compose.yaml | 31 +++++ test/engine/cleanup/helper.go | 1 + test/engine/cleanup/monitor.go | 11 ++ test/engine/setup/monitor/monitor.go | 97 ++++++++++++++ test/engine/setup/monitor/statistic.go | 61 +++++++++ test/engine/trigger/file.go | 48 +++++++ test/go.mod | 32 ++--- test/go.sum | 51 +++++--- 13 files changed, 530 insertions(+), 35 deletions(-) create mode 100644 test/benchmark/benchmark_docker_test.go create mode 100644 test/benchmark/main_test.go create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml create mode 100644 test/engine/cleanup/monitor.go create mode 100644 test/engine/setup/monitor/monitor.go create mode 100644 test/engine/setup/monitor/statistic.go create mode 100644 test/engine/trigger/file.go diff --git a/licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md b/licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md index 6a2ea80e81..b82923e98c 100644 --- a/licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md +++ b/licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md @@ -16,7 +16,6 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion - [github.com/opencontainers/go-digest](https://pkg.go.dev/github.com/opencontainers/go-digest?tab=licenses) - [github.com/opencontainers/image-spec](https://pkg.go.dev/github.com/opencontainers/image-spec?tab=licenses) - [github.com/opencontainers/runc](https://pkg.go.dev/github.com/opencontainers/runc?tab=licenses) -- [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto?tab=licenses) - [google.golang.org/grpc](https://pkg.go.dev/google.golang.org/grpc?tab=licenses) - [gopkg.in/ini.v1](https://pkg.go.dev/gopkg.in/ini.v1?tab=licenses) - [gopkg.in/yaml.v2](https://pkg.go.dev/gopkg.in/yaml.v2?tab=licenses) @@ -46,6 +45,8 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion - [sigs.k8s.io/json](https://pkg.go.dev/sigs.k8s.io/json?tab=licenses) - [sigs.k8s.io/structured-merge-diff](https://pkg.go.dev/sigs.k8s.io/structured-merge-diff?tab=licenses) - [sigs.k8s.io/yaml](https://pkg.go.dev/sigs.k8s.io/yaml?tab=licenses) +- [github.com/google/cadvisor](https://pkg.go.dev/github.com/google/cadvisor?tab=licenses) +- [google.golang.org/genproto/googleapis/rpc](https://pkg.go.dev/google.golang.org/genproto/googleapis/rpc?tab=licenses) ## BSD licenses diff --git a/test/benchmark/benchmark_docker_test.go b/test/benchmark/benchmark_docker_test.go new file mode 100644 index 0000000000..9a560bce68 --- /dev/null +++ b/test/benchmark/benchmark_docker_test.go @@ -0,0 +1,68 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package e2e + +import ( + "testing" + + "github.com/cucumber/godog" +) + +func TestE2EOnDockerCompose(t *testing.T) { + suite := godog.TestSuite{ + Name: "E2EOnDockerCompose", + ScenarioInitializer: scenarioInitializer, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"test_cases"}, + Tags: "@e2e && @docker-compose && ~@ebpf", + TestingT: t, + }, + } + if suite.Run() != 0 { + t.Fail() + } +} + +func TestE2EOnDockerComposeCore(t *testing.T) { + suite := godog.TestSuite{ + Name: "E2EOnDockerCompose", + ScenarioInitializer: scenarioInitializer, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"test_cases"}, + Tags: "@e2e-core && @docker-compose && ~@ebpf", + TestingT: t, + }, + } + if suite.Run() != 0 { + t.Fail() + } +} + +func TestE2EOnDockerComposePerformance(t *testing.T) { + suite := godog.TestSuite{ + Name: "E2EOnDockerCompose", + ScenarioInitializer: scenarioInitializer, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"test_cases"}, + Tags: "@e2e-performance && @docker-compose && ~@ebpf", + TestingT: t, + }, + } + if suite.Run() != 0 { + t.Fail() + } +} diff --git a/test/benchmark/main_test.go b/test/benchmark/main_test.go new file mode 100644 index 0000000000..1fd5eab4b2 --- /dev/null +++ b/test/benchmark/main_test.go @@ -0,0 +1,123 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package e2e + +import ( + "context" + "os" + "strconv" + "testing" + "time" + + "github.com/cucumber/godog" + + "github.com/alibaba/ilogtail/pkg/logger" + "github.com/alibaba/ilogtail/test/config" + "github.com/alibaba/ilogtail/test/engine/cleanup" + "github.com/alibaba/ilogtail/test/engine/control" + "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" + "github.com/alibaba/ilogtail/test/engine/setup/subscriber" + "github.com/alibaba/ilogtail/test/engine/trigger" + "github.com/alibaba/ilogtail/test/engine/verify" +) + +func TestMain(m *testing.M) { + loggerOptions := []logger.ConfigOption{ + logger.OptionAsyncLogger, + } + loggerOptions = append(loggerOptions, logger.OptionInfoLevel) + logger.InitTestLogger(loggerOptions...) + + config.TestConfig = config.Config{} + // Log + config.TestConfig.GeneratedLogDir = os.Getenv("GENERATED_LOG_DIR") + if len(config.TestConfig.GeneratedLogDir) == 0 { + config.TestConfig.GeneratedLogDir = "/tmp/ilogtail" + } + config.TestConfig.WorkDir = os.Getenv("WORK_DIR") + + // SSH + config.TestConfig.SSHUsername = os.Getenv("SSH_USERNAME") + config.TestConfig.SSHIP = os.Getenv("SSH_IP") + config.TestConfig.SSHPassword = os.Getenv("SSH_PASSWORD") + + // K8s + config.TestConfig.KubeConfigPath = os.Getenv("KUBE_CONFIG_PATH") + + // SLS + config.TestConfig.Project = os.Getenv("PROJECT") + config.TestConfig.Logstore = os.Getenv("LOGSTORE") + config.TestConfig.AccessKeyID = os.Getenv("ACCESS_KEY_ID") + config.TestConfig.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET") + config.TestConfig.Endpoint = os.Getenv("ENDPOINT") + config.TestConfig.Aliuid = os.Getenv("ALIUID") + config.TestConfig.QueryEndpoint = os.Getenv("QUERY_ENDPOINT") + config.TestConfig.Region = os.Getenv("REGION") + timeout, err := strconv.ParseInt(os.Getenv("RETRY_TIMEOUT"), 10, 64) + if err != nil { + timeout = 60 + } + config.TestConfig.RetryTimeout = time.Duration(timeout) * time.Second + code := m.Run() + logger.Flush() + os.Exit(code) +} + +func scenarioInitializer(ctx *godog.ScenarioContext) { + // Given + ctx.Given(`^\{(\S+)\} environment$`, setup.InitEnv) + ctx.Given(`^iLogtail depends on containers \{(.*)\}`, setup.SetDockerComposeDependOn) + ctx.Given(`^iLogtail container mount \{(.*)\} to \{(.*)\}`, setup.MountVolume) + ctx.Given(`^iLogtail expose port \{(.*)\} to \{(.*)\}`, setup.ExposePort) + ctx.Given(`^\{(.*)\} local config as below`, control.AddLocalConfig) + ctx.Given(`^\{(.*)\} http config as below`, control.AddHTTPConfig) + ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig) + ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) + + // When + ctx.When(`^generate \{(\d+)\} regex logs, with interval \{(\d+)\}ms$`, trigger.RegexSingle) + ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) + ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) + ctx.When(`^add k8s label \{(.*)\}`, control.AddLabel) + ctx.When(`^remove k8s label \{(.*)\}`, control.RemoveLabel) + ctx.When(`^start docker-compose \{(\S+)\}`, setup.StartDockerComposeEnv) + ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) + + // Then + ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount) + ctx.Then(`^there is at least \{(\d+)\} logs$`, verify.LogCountAtLeast) + ctx.Then(`^there is at least \{(\d+)\} logs with filter key \{(.*)\} value \{(.*)\}$`, verify.LogCountAtLeastWithFilter) + ctx.Then(`^the log fields match regex single`, verify.RegexSingle) + ctx.Then(`^the log fields match kv`, verify.LogFieldKV) + ctx.Then(`^the log tags match kv`, verify.TagKV) + ctx.Then(`^the context of log is valid$`, verify.LogContext) + ctx.Then(`^the log fields match`, verify.LogField) + ctx.Then(`^the log labels match`, verify.LogLabel) + ctx.Then(`^the logtail log contains \{(\d+)\} times of \{(.*)\}$`, verify.LogtailPluginLog) + ctx.Then(`wait \{(\d+)\} seconds`, func(ctx context.Context, t int) context.Context { + time.Sleep(time.Duration(t) * time.Second) + return ctx + }) + + // Cleanup + ctx.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { + cleanup.HandleSignal() + return ctx, nil + }) + ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { + cleanup.All() + return ctx, nil + }) +} diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature new file mode 100644 index 0000000000..763d9d2ed3 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -0,0 +1,39 @@ +@input +Feature: performance file to blackhole iLogtail + Performance file to blackhole iLogtail + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeiLogtail + Given {docker-compose} environment + Given subcribe data from {grpc} with config + """ + """ + Given {performance-file-to-blackhole-ilogtail-case} local config as below + """ + enable: true + inputs: + - Type: input_file + FilePaths: + - /home/test-log/json.log + processors: + - Type: processor_parse_json_native + SourceKey: content + - Type: processor_filter_regex_native + FilterKey: + - user-agent + FilterRegex: + - ^no-agent$ + flushers: + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore + """ + Given iLogtail container mount {./a.log} to {/home/test-log/json.log} + When start docker-compose {performance_file_to_blackhole_ilogtail} + When start monitor {e2e-ilogtailC-1} + When generate logs to file, speed {10}MB/s, total {3}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml new file mode 100644 index 0000000000..49bd5213c5 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + ports: + - "8080:8080" + privileged: true + devices: + - /dev/kmsg + restart: unless-stopped diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index 39562e493e..6fd81f68f7 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -46,6 +46,7 @@ func All() { _, _ = control.RemoveAllLocalConfig(ctx) _, _ = AllGeneratedLog(ctx) _, _ = GoTestCache(ctx) + _, _ = StopMonitor(ctx) _, _ = DeleteContainers(ctx) if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() diff --git a/test/engine/cleanup/monitor.go b/test/engine/cleanup/monitor.go new file mode 100644 index 0000000000..b541b12850 --- /dev/null +++ b/test/engine/cleanup/monitor.go @@ -0,0 +1,11 @@ +package cleanup + +import ( + "context" + + "github.com/alibaba/ilogtail/test/engine/setup/monitor" +) + +func StopMonitor(ctx context.Context) (context.Context, error) { + return monitor.StopMonitor(ctx) +} diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go new file mode 100644 index 0000000000..570fb364f4 --- /dev/null +++ b/test/engine/setup/monitor/monitor.go @@ -0,0 +1,97 @@ +package monitor + +import ( + "context" + "fmt" + "log" + "os" + "sync/atomic" + "time" + + "github.com/google/cadvisor/client" + v1 "github.com/google/cadvisor/info/v1" + + "github.com/alibaba/ilogtail/test/config" +) + +const ( + cadvisorURL = "http://localhost:8080/" + interval = 3 +) + +var stopCh chan bool +var isMonitoring atomic.Bool + +func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { + stopCh = make(chan bool) + isMonitoring.Store(true) + // connect to cadvisor + client, err := client.NewClient("http://localhost:8080/") + if err != nil { + return ctx, err + } + // 获取机器信息 + _, err = client.MachineInfo() + if err != nil { + // 处理错误 + return ctx, err + } + // fmt.Println("Machine Info:", machineInfo) + fmt.Println("Start monitoring container:", containerName) + go monitoring(client, containerName) + return ctx, nil +} + +func StopMonitor(ctx context.Context) (context.Context, error) { + if isMonitoring.Load() { + stopCh <- true + } + return ctx, nil +} + +func monitoring(client *client.Client, containerName string) { + // create csv file + reportDir := config.CaseHome + "/report/" + if _, err := os.Stat(reportDir); os.IsNotExist(err) { + // 文件夹不存在，创建文件夹 + err := os.MkdirAll(reportDir, 0755) // 使用适当的权限 + if err != nil { + log.Fatalf("Failed to create folder: %s", err) + } + } + file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + fmt.Println("Error creating file:", err) + return + } + defer file.Close() + header := "CPU Usage Max(%),CPU Usage Avg(%), Memory Usage Max(MB), Memory Usage Avg(MB)\n" + _, err = file.WriteString(header) + if err != nil { + fmt.Println("Error writring file header:", err) + return + } + // new ticker + ticker := time.NewTicker(interval * time.Second) + defer ticker.Stop() + // read from cadvisor per interval seconds + request := &v1.ContainerInfoRequest{NumStats: 10} + monitorStatistic := NewMonitorStatistic() + for { + select { + case <-stopCh: + fmt.Fprintln(file, monitorStatistic.cpu.maxVal, ",", monitorStatistic.cpu.avgVal, ",", monitorStatistic.mem.maxVal, ",", monitorStatistic.mem.avgVal) + return + case <-ticker.C: + // 获取容器信息 + containerInfo, err := client.DockerContainer(containerName, request) + if err != nil { + fmt.Println("Error getting container info:", err) + return + } + for _, stat := range containerInfo.Stats { + monitorStatistic.UpdateStatistic(stat) + } + } + } +} diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go new file mode 100644 index 0000000000..bacc06737b --- /dev/null +++ b/test/engine/setup/monitor/statistic.go @@ -0,0 +1,61 @@ +package monitor + +import ( + "fmt" + + v1 "github.com/google/cadvisor/info/v1" +) + +type Statistic struct { + maxVal float64 + avgVal float64 + cnt int64 +} + +func (s *Statistic) Add(val float64) { + if s.cnt < 1 { + s.maxVal = val + s.avgVal = val + s.cnt++ + return + } + if s.maxVal < val { + s.maxVal = val + } + s.avgVal = float64(s.cnt)/float64(s.cnt+1)*s.avgVal + val/float64(s.cnt+1) + s.cnt++ +} + +type MonitorStatistic struct { + cpu Statistic + mem Statistic + lastStat *v1.ContainerStats +} + +func NewMonitorStatistic() *MonitorStatistic { + return &MonitorStatistic{ + cpu: Statistic{}, + mem: Statistic{}, + lastStat: nil, + } +} + +func (m *MonitorStatistic) UpdateStatistic(stat *v1.ContainerStats) { + if m.lastStat != nil && !stat.Timestamp.After(m.lastStat.Timestamp) { + return + } + cpuUsageRateTotal := calculateCpuUsageRate(m.lastStat, stat) + m.cpu.Add(cpuUsageRateTotal) + m.mem.Add(float64(stat.Memory.Usage) / 1024 / 1024) + m.lastStat = stat + fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) +} + +func calculateCpuUsageRate(lastStat, stat *v1.ContainerStats) float64 { + if lastStat == nil { + return 0 + } + cpuUsageTotal := stat.Cpu.Usage.Total - lastStat.Cpu.Usage.Total + cpuUsageRateTotal := float64(cpuUsageTotal) * 100 / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds()) + return cpuUsageRateTotal +} diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go new file mode 100644 index 0000000000..d890044fc8 --- /dev/null +++ b/test/engine/trigger/file.go @@ -0,0 +1,48 @@ +package trigger + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alibaba/ilogtail/test/config" + "golang.org/x/time/rate" +) + +func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, templateStr string) (context.Context, error) { + // 打开或创建文件 + if !filepath.IsAbs(path) { + path = filepath.Join(config.CaseHome, path) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return ctx, err + } + defer file.Close() + + // interval = templateLen / speed + interval := time.Microsecond * time.Duration(len(templateStr)/speed) + + limiter := rate.NewLimiter(rate.Every(interval), 1) + + // 总时间控制 + timeout := time.After(time.Minute * time.Duration(totalTime)) + + for { + select { + case <-ctx.Done(): // 上下文取消 + return ctx, ctx.Err() + case <-timeout: // 总时间到 + return ctx, nil + default: + if err := limiter.Wait(ctx); err != nil { + return ctx, err + } + if _, err := fmt.Fprintln(file, templateStr); err != nil { + return ctx, err + } + } + } +} diff --git a/test/go.mod b/test/go.mod index 57205422d6..65fed5531d 100644 --- a/test/go.mod +++ b/test/go.mod @@ -10,15 +10,17 @@ require ( github.com/alibabacloud-go/tea v1.2.1 github.com/avast/retry-go/v4 v4.6.0 github.com/cucumber/godog v0.14.1 - github.com/docker/docker v20.10.23+incompatible + github.com/docker/docker v20.10.27+incompatible github.com/docker/go-connections v0.4.0 github.com/elastic/go-elasticsearch/v8 v8.6.0 + github.com/google/cadvisor v0.49.1 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/melbahja/goph v1.4.0 github.com/mitchellh/mapstructure v1.5.0 github.com/testcontainers/testcontainers-go v0.14.0 - golang.org/x/crypto v0.10.0 - google.golang.org/grpc v1.53.0 + golang.org/x/crypto v0.16.0 + golang.org/x/time v0.3.0 + google.golang.org/grpc v1.58.3 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.23.4 k8s.io/apimachinery v0.23.4 @@ -53,7 +55,7 @@ require ( github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -79,13 +81,13 @@ require ( github.com/moby/spdystream v0.2.0 // indirect github.com/moby/sys/mount v0.3.3 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect - github.com/opencontainers/runc v1.1.3 // indirect + github.com/opencontainers/runc v1.1.12 // indirect github.com/paulmach/orb v0.8.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect @@ -99,21 +101,20 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.11.2 // indirect go.opentelemetry.io/otel/trace v1.11.2 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.30.0 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect @@ -122,4 +123,5 @@ require ( replace ( github.com/alibaba/ilogtail => ../ github.com/alibaba/ilogtail/pkg => ../pkg + github.com/google/cadvisor v0.49.1 => github.com/google/cadvisor v0.49.0 ) diff --git a/test/go.sum b/test/go.sum index 3b1bada07b..5077263740 100644 --- a/test/go.sum +++ b/test/go.sum @@ -184,6 +184,7 @@ github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -347,8 +348,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= github.com/cucumber/godog v0.14.1 h1:HGZhcOyyfaKclHjJ+r/q93iaTJZLKYW6Tv3HkmUE6+M= @@ -377,12 +378,13 @@ github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.23+incompatible h1:1ZQUUYAdh+oylOT85aA2ZcfRp22jmLhoaEcVEfK8dyA= -github.com/docker/docker v20.10.23+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.27+incompatible h1:Id/ZooynV4ZlD6xX20RCd3SR0Ikn7r4QZDa2ECK2TgA= +github.com/docker/docker v20.10.27+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -539,6 +541,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cadvisor v0.49.0 h1:1PYeiORXmcFYi609M4Qvq5IzcvcVaWgYxDt78uH8jYA= +github.com/google/cadvisor v0.49.0/go.mod h1:s6Fqwb2KiWG6leCegVhw4KW40tf9f7m+SF1aXiE8Wsk= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -753,8 +757,9 @@ github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGq github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -826,8 +831,9 @@ github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84 github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= +github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1098,8 +1104,9 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1201,8 +1208,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1216,8 +1224,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1341,8 +1349,9 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1351,8 +1360,9 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1364,8 +1374,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1530,8 +1541,8 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1561,8 +1572,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1677,8 +1688,9 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= @@ -1690,8 +1702,9 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 762a3f3b5af9b5ff69fd6c3f226867edd3b32db4 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Thu, 11 Jul 2024 13:08:16 +0800 Subject: [PATCH 02/88] fix lint check errors --- test/engine/setup/monitor/monitor.go | 22 +++++++++++++++++----- test/engine/setup/monitor/statistic.go | 24 ++++++++++++------------ test/engine/trigger/file.go | 9 +++++++-- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 570fb364f4..26a4677146 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "path/filepath" "sync/atomic" "time" @@ -51,20 +52,28 @@ func StopMonitor(ctx context.Context) (context.Context, error) { func monitoring(client *client.Client, containerName string) { // create csv file - reportDir := config.CaseHome + "/report/" - if _, err := os.Stat(reportDir); os.IsNotExist(err) { + reportDir := filepath.Join(config.CaseHome, "report") + reportDir, err := filepath.Abs(reportDir) + if err != nil { + log.Fatalf("Failed to get absolute path: %s", err) + } + if _, err = os.Stat(reportDir); os.IsNotExist(err) { // 文件夹不存在，创建文件夹 - err := os.MkdirAll(reportDir, 0755) // 使用适当的权限 + err := os.MkdirAll(reportDir, 0750) // 使用适当的权限 if err != nil { log.Fatalf("Failed to create folder: %s", err) } } - file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + reportDir = filepath.Join(reportDir, "performance.csv") + reportDir, err = filepath.Abs(reportDir) + if err != nil { + log.Fatalf("Failed to get absolute path: %s", err) + } + file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) if err != nil { fmt.Println("Error creating file:", err) return } - defer file.Close() header := "CPU Usage Max(%),CPU Usage Avg(%), Memory Usage Max(MB), Memory Usage Avg(MB)\n" _, err = file.WriteString(header) if err != nil { @@ -81,6 +90,9 @@ func monitoring(client *client.Client, containerName string) { select { case <-stopCh: fmt.Fprintln(file, monitorStatistic.cpu.maxVal, ",", monitorStatistic.cpu.avgVal, ",", monitorStatistic.mem.maxVal, ",", monitorStatistic.mem.avgVal) + if err = file.Close(); err != nil { + fmt.Println("Error closing file:", err) + } return case <-ticker.C: // 获取容器信息 diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index bacc06737b..be3494d5bf 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -6,13 +6,13 @@ import ( v1 "github.com/google/cadvisor/info/v1" ) -type Statistic struct { +type Info struct { maxVal float64 avgVal float64 cnt int64 } -func (s *Statistic) Add(val float64) { +func (s *Info) Add(val float64) { if s.cnt < 1 { s.maxVal = val s.avgVal = val @@ -26,32 +26,32 @@ func (s *Statistic) Add(val float64) { s.cnt++ } -type MonitorStatistic struct { - cpu Statistic - mem Statistic +type Statistic struct { + cpu Info + mem Info lastStat *v1.ContainerStats } -func NewMonitorStatistic() *MonitorStatistic { - return &MonitorStatistic{ - cpu: Statistic{}, - mem: Statistic{}, +func NewMonitorStatistic() *Statistic { + return &Statistic{ + cpu: Info{}, + mem: Info{}, lastStat: nil, } } -func (m *MonitorStatistic) UpdateStatistic(stat *v1.ContainerStats) { +func (m *Statistic) UpdateStatistic(stat *v1.ContainerStats) { if m.lastStat != nil && !stat.Timestamp.After(m.lastStat.Timestamp) { return } - cpuUsageRateTotal := calculateCpuUsageRate(m.lastStat, stat) + cpuUsageRateTotal := calculateCPUUsageRate(m.lastStat, stat) m.cpu.Add(cpuUsageRateTotal) m.mem.Add(float64(stat.Memory.Usage) / 1024 / 1024) m.lastStat = stat fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) } -func calculateCpuUsageRate(lastStat, stat *v1.ContainerStats) float64 { +func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { if lastStat == nil { return 0 } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index d890044fc8..97a890a6ae 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -16,11 +16,10 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t if !filepath.IsAbs(path) { path = filepath.Join(config.CaseHome, path) } - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) if err != nil { return ctx, err } - defer file.Close() // interval = templateLen / speed interval := time.Microsecond * time.Duration(len(templateStr)/speed) @@ -33,8 +32,14 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t for { select { case <-ctx.Done(): // 上下文取消 + if err := file.Close(); err != nil { + return ctx, err + } return ctx, ctx.Err() case <-timeout: // 总时间到 + if err := file.Close(); err != nil { + return ctx, err + } return ctx, nil default: if err := limiter.Wait(ctx); err != nil { From 6078a372c3d6406d63e1f5e9e61815dca196efbf Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Thu, 11 Jul 2024 16:56:57 +0800 Subject: [PATCH 03/88] add file-to-blackhole performance test --- test/benchmark/main_test.go | 1 + .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 32 +++ .../filebeat.yml | 17 ++ .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 14 ++ .../main.conf | 10 + .../parsers.conf | 3 + .../case.feature | 8 +- .../docker-compose.yaml | 2 +- .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 9 + .../vector.yaml | 11 + test/engine/cleanup/helper.go | 1 + .../setup/controller/docker_compose_boot.go | 4 +- test/engine/setup/docker_compose.go | 8 +- test/engine/setup/dockercompose/common.go | 11 +- .../setup/dockercompose/compose benchmark.go | 188 ++++++++++++++++++ test/engine/setup/monitor/monitor.go | 42 ++-- test/engine/setup/monitor/statistic.go | 4 +- test/engine/trigger/file.go | 40 ++-- 24 files changed, 387 insertions(+), 60 deletions(-) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml create mode 100644 test/engine/setup/dockercompose/compose benchmark.go diff --git a/test/benchmark/main_test.go b/test/benchmark/main_test.go index 1fd5eab4b2..fd1e68b501 100644 --- a/test/benchmark/main_test.go +++ b/test/benchmark/main_test.go @@ -78,6 +78,7 @@ func TestMain(m *testing.M) { func scenarioInitializer(ctx *godog.ScenarioContext) { // Given ctx.Given(`^\{(\S+)\} environment$`, setup.InitEnv) + ctx.Given(`^docker-compose type \{(\S+)\}$`, setup.SetDockerComposeType) ctx.Given(`^iLogtail depends on containers \{(.*)\}`, setup.SetDockerComposeDependOn) ctx.Given(`^iLogtail container mount \{(.*)\} to \{(.*)\}`, setup.MountVolume) ctx.Given(`^iLogtail expose port \{(.*)\} to \{(.*)\}`, setup.ExposePort) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature new file mode 100644 index 0000000000..5285082080 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole filebeat + Performance file to blackhole filebeat + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeFilebeat + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_filebeat} + When start monitor {e2e-filebeat-1} + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml new file mode 100644 index 0000000000..8e12d4a340 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -0,0 +1,32 @@ +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.2 + environment: + - ELASTIC_PASSWORD=elastic + - discovery.type=single-node + - xpack.security.http.ssl.enabled=false + - xpack.license.self_generated.type=trial + ports: + - 9200:9200 + healthcheck: + test: ["CMD-SHELL", "curl -u elastic:elastic -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] + interval: 10s + timeout: 5s + retries: 3 + restart: always + + filebeat: + image: docker.elastic.co/beats/filebeat:8.14.2 + user: root + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ./a.log:/home/filebeat/a.log:ro + command: filebeat -e --strict.perms=false + environment: + - OUTPUT_ELASTICSEARCH_HOSTS=["elasticsearch:9200"] + depends_on: + elasticsearch: + condition: service_healthy + restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml new file mode 100644 index 0000000000..1c95c5ec77 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -0,0 +1,17 @@ +filebeat.inputs: +- type: filestream + id: input-file + paths: + - /home/filebeat/a.log + +processors: + - decode_json_fields: + fields: ["message"] + process_array: false + max_depth: 1 + target: "" + overwrite_keys: false + add_error_key: true + +output.discard: + enabled: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature new file mode 100644 index 0000000000..a29fac66f8 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole fluentbit + Performance file to blackhole fluentbit + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeFluentbit + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_fluentbit} + When start monitor {e2e-fluent-bit-1} + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml new file mode 100644 index 0000000000..aace706b7b --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + + fluent-bit: + image: cr.fluentbit.io/fluent/fluent-bit + command: ["-c", "/tmp/main.conf"] + volumes: + - ./main.conf:/tmp/main.conf + - ./parsers.conf:/tmp/parsers.conf + - ./a.log:/home/fluentbit/a.log + restart: always + + diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf new file mode 100644 index 0000000000..baaf715b82 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -0,0 +1,10 @@ +[SERVICE] + Parsers+File /tmp/parsers.conf + +[INPUT] + Name tail + Path /home/fluentbit/a.log + +[OUTPUT] + Name stdout + Match non diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf new file mode 100644 index 0000000000..27cc2efc00 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf @@ -0,0 +1,3 @@ +[PARSER] + Name docker + Format json \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 763d9d2ed3..eceb6534fc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -5,9 +5,7 @@ Feature: performance file to blackhole iLogtail @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeiLogtail Given {docker-compose} environment - Given subcribe data from {grpc} with config - """ - """ + Given docker-compose type {e2e} Given {performance-file-to-blackhole-ilogtail-case} local config as below """ enable: true @@ -33,7 +31,7 @@ Feature: performance file to blackhole iLogtail Given iLogtail container mount {./a.log} to {/home/test-log/json.log} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {e2e-ilogtailC-1} - When generate logs to file, speed {10}MB/s, total {3}min, to file {./a.log}, template + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ \ No newline at end of file + """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 49bd5213c5..df9ceffb57 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -28,4 +28,4 @@ services: privileged: true devices: - /dev/kmsg - restart: unless-stopped + restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature new file mode 100644 index 0000000000..b6265805d4 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole vector + Performance file to blackhole vector + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeVector + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_vector} + When start monitor {e2e-vector-1} + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml new file mode 100644 index 0000000000..3763d9a3e6 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -0,0 +1,9 @@ +version: '3.8' + +services: + vector: + image: timberio/vector:0.39.0-debian + volumes: + - ./vector.yaml:/etc/vector/vector.yaml + - ./a.log:/home/vector-log/a.log + restart: always \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml new file mode 100644 index 0000000000..cd13fc3489 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml @@ -0,0 +1,11 @@ +sources: + input_file: + type: file + include: + - /home/vector-log/*.log + +sinks: + my_sink_id: + type: blackhole + inputs: + - input_file \ No newline at end of file diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index 6fd81f68f7..06827823f8 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -48,6 +48,7 @@ func All() { _, _ = GoTestCache(ctx) _, _ = StopMonitor(ctx) _, _ = DeleteContainers(ctx) + // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() } diff --git a/test/engine/setup/controller/docker_compose_boot.go b/test/engine/setup/controller/docker_compose_boot.go index 1ffbc83636..394affe08d 100644 --- a/test/engine/setup/controller/docker_compose_boot.go +++ b/test/engine/setup/controller/docker_compose_boot.go @@ -27,9 +27,9 @@ import ( type BootController struct { } -func (c *BootController) Init() error { +func (c *BootController) Init(initType string) error { logger.Info(context.Background(), "boot controller is initializing....") - return dockercompose.Load() + return dockercompose.Load(initType) } func (c *BootController) Start(ctx context.Context) error { diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 202a4c6230..cb4e51fcb0 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -16,10 +16,16 @@ import ( const dependencyHome = "test_cases" +var dockerComposeType = "e2e" + type DockerComposeEnv struct { BootController *controller.BootController } +func SetDockerComposeType(t string) { + dockerComposeType = t +} + func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context.Context, error) { if dockerComposeEnv, ok := Env.(*DockerComposeEnv); ok { path := dependencyHome + "/" + dependencyName @@ -28,7 +34,7 @@ func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context. return ctx, err } dockerComposeEnv.BootController = new(controller.BootController) - if err = dockerComposeEnv.BootController.Init(); err != nil { + if err = dockerComposeEnv.BootController.Init(dockerComposeType); err != nil { return ctx, err } diff --git a/test/engine/setup/dockercompose/common.go b/test/engine/setup/dockercompose/common.go index f9662ef8d7..9f35c3b90e 100644 --- a/test/engine/setup/dockercompose/common.go +++ b/test/engine/setup/dockercompose/common.go @@ -33,10 +33,17 @@ type Booter interface { } // Load configuration to the Booter. -func Load() error { +func Load(loadType string) error { mu.Lock() defer mu.Unlock() - instance = NewComposeBooter() + switch loadType { + case "e2e": + instance = NewComposeBooter() + case "benchmark": + instance = NewComposeBenchmarkBooter() + default: + return errors.New("invalid load type") + } return nil } diff --git a/test/engine/setup/dockercompose/compose benchmark.go b/test/engine/setup/dockercompose/compose benchmark.go new file mode 100644 index 0000000000..4b566c10a3 --- /dev/null +++ b/test/engine/setup/dockercompose/compose benchmark.go @@ -0,0 +1,188 @@ +// Copyright 2021 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dockercompose + +import ( + "context" + "os" + + "github.com/docker/docker/api/types" + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/testcontainers/testcontainers-go" + "gopkg.in/yaml.v3" + + "github.com/alibaba/ilogtail/pkg/logger" + "github.com/alibaba/ilogtail/test/config" +) + +const ( + cadvisorTemplate = `version: '3.8' +services: + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + ports: + - "8080:8080" + privileged: true + devices: + - /dev/kmsg + restart: unless-stopped +` +) + +// ComposeBooter control docker-compose to start or stop containers. +type ComposeBenchmarkBooter struct { + cli *client.Client + cadvisorID string +} + +// NewComposeBooter create a new compose booter. +func NewComposeBenchmarkBooter() *ComposeBenchmarkBooter { + return &ComposeBenchmarkBooter{} +} + +func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { + if err := c.createComposeFile(); err != nil { + return err + } + compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).WithCommand([]string{"up", "-d", "--build"}) + strategyWrappers := withExposedService(compose) + execError := compose.Invoke() + if execError.Error != nil { + logger.Error(context.Background(), "START_DOCKER_COMPOSE_ERROR", + "stdout", execError.Error.Error()) + return execError.Error + } + cli, err := CreateDockerClient() + if err != nil { + return err + } + c.cli = cli + + list, err := cli.ContainerList(context.Background(), types.ContainerListOptions{ + Filters: filters.NewArgs(filters.Arg("name", "e2e-cadvisor")), + }) + if len(list) != 1 { + logger.Errorf(context.Background(), "CADVISOR_COMPOSE_ALARM", "cadvisor container size is not equal 1, got %d count", len(list)) + return err + } + c.cadvisorID = list[0].ID + + // the docker engine cannot access host on the linux platform, more details please see: https://github.com/docker/for-linux/issues/264 + cmd := []string{ + "sh", + "-c", + "env |grep HOST_OS|grep Linux && ip -4 route list match 0/0|awk '{print $3\" host.docker.internal\"}' >> /etc/hosts", + } + if err = c.exec(c.cadvisorID, cmd); err != nil { + return err + } + err = registerDockerNetMapping(strategyWrappers) + logger.Debugf(context.Background(), "registered net mapping: %v", networkMapping) + return err +} + +func (c *ComposeBenchmarkBooter) Stop() error { + execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).Down() + if execError.Error != nil { + logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", + "stdout", execError.Stdout.Error(), "stderr", execError.Stderr.Error()) + return execError.Error + } + _ = os.Remove(config.CaseHome + finalFileName) + return nil +} + +func (c *ComposeBenchmarkBooter) exec(id string, cmd []string) error { + cfg := dockertypes.ExecConfig{ + User: "root", + Cmd: cmd, + } + resp, err := c.cli.ContainerExecCreate(context.Background(), id, cfg) + if err != nil { + logger.Errorf(context.Background(), "DOCKER_EXEC_ALARM", "cannot create exec config: %v", err) + return err + } + err = c.cli.ContainerExecStart(context.Background(), resp.ID, dockertypes.ExecStartCheck{ + Detach: false, + Tty: false, + }) + if err != nil { + logger.Errorf(context.Background(), "DOCKER_EXEC_ALARM", "cannot start exec config: %v", err) + return err + } + return nil +} + +func (c *ComposeBenchmarkBooter) CopyCoreLogs() { +} + +func (c *ComposeBenchmarkBooter) createComposeFile() error { + // read the case docker compose file. + if _, err := os.Stat(config.CaseHome); os.IsNotExist(err) { + if err = os.MkdirAll(config.CaseHome, 0750); err != nil { + return err + } + } + _, err := os.Stat(config.CaseHome + config.DockerComposeFileName) + var bytes []byte + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + if bytes, err = os.ReadFile(config.CaseHome + config.DockerComposeFileName); err != nil { + return err + } + } + cfg := c.getAdvisorConfig() + services := cfg["services"].(map[string]interface{}) + // merge docker compose file. + if len(bytes) > 0 { + caseCfg := make(map[string]interface{}) + if err = yaml.Unmarshal(bytes, &caseCfg); err != nil { + return err + } + newServices := caseCfg["services"].(map[string]interface{}) + for k := range newServices { + services[k] = newServices[k] + } + } + yml, err := yaml.Marshal(cfg) + if err != nil { + return err + } + return os.WriteFile(config.CaseHome+finalFileName, yml, 0600) +} + +// getLogtailpluginConfig find the docker compose configuration of the ilogtail. +func (c *ComposeBenchmarkBooter) getAdvisorConfig() map[string]interface{} { + cfg := make(map[string]interface{}) + f, _ := os.Create(config.CoverageFile) + _ = f.Close() + if err := yaml.Unmarshal([]byte(cadvisorTemplate), &cfg); err != nil { + panic(err) + } + bytes, _ := yaml.Marshal(cfg) + println(string(bytes)) + return cfg +} diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 26a4677146..3d65521df6 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -6,6 +6,7 @@ import ( "log" "os" "path/filepath" + "strings" "sync/atomic" "time" @@ -24,20 +25,20 @@ var stopCh chan bool var isMonitoring atomic.Bool func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { - stopCh = make(chan bool) - isMonitoring.Store(true) // connect to cadvisor client, err := client.NewClient("http://localhost:8080/") if err != nil { return ctx, err } // 获取机器信息 - _, err = client.MachineInfo() + request := &v1.ContainerInfoRequest{NumStats: 10} + _, err = client.DockerContainer(containerName, request) if err != nil { - // 处理错误 + fmt.Println("Error getting container info:", err) return ctx, err } - // fmt.Println("Machine Info:", machineInfo) + stopCh = make(chan bool) + isMonitoring.Store(true) fmt.Println("Start monitoring container:", containerName) go monitoring(client, containerName) return ctx, nil @@ -59,27 +60,11 @@ func monitoring(client *client.Client, containerName string) { } if _, err = os.Stat(reportDir); os.IsNotExist(err) { // 文件夹不存在，创建文件夹 - err := os.MkdirAll(reportDir, 0750) // 使用适当的权限 + err = os.MkdirAll(reportDir, 0750) if err != nil { log.Fatalf("Failed to create folder: %s", err) } } - reportDir = filepath.Join(reportDir, "performance.csv") - reportDir, err = filepath.Abs(reportDir) - if err != nil { - log.Fatalf("Failed to get absolute path: %s", err) - } - file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - fmt.Println("Error creating file:", err) - return - } - header := "CPU Usage Max(%),CPU Usage Avg(%), Memory Usage Max(MB), Memory Usage Avg(MB)\n" - _, err = file.WriteString(header) - if err != nil { - fmt.Println("Error writring file header:", err) - return - } // new ticker ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() @@ -89,9 +74,16 @@ func monitoring(client *client.Client, containerName string) { for { select { case <-stopCh: - fmt.Fprintln(file, monitorStatistic.cpu.maxVal, ",", monitorStatistic.cpu.avgVal, ",", monitorStatistic.mem.maxVal, ",", monitorStatistic.mem.avgVal) - if err = file.Close(); err != nil { - fmt.Println("Error closing file:", err) + isMonitoring.Store(false) + var builder strings.Builder + builder.WriteString("Metric,Value\n") + builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Max(%)", monitorStatistic.cpu.maxVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Avg(%)", monitorStatistic.cpu.avgVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Max(MB)", monitorStatistic.mem.maxVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Avg(MB)", monitorStatistic.mem.avgVal)) + err = os.WriteFile(filepath.Join(reportDir, "monitor.csv"), []byte(builder.String()), 0600) + if err != nil { + log.Default().Printf("Failed to write monitor result: %s", err) } return case <-ticker.C: diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index be3494d5bf..442628dfe6 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -1,8 +1,6 @@ package monitor import ( - "fmt" - v1 "github.com/google/cadvisor/info/v1" ) @@ -48,7 +46,7 @@ func (m *Statistic) UpdateStatistic(stat *v1.ContainerStats) { m.cpu.Add(cpuUsageRateTotal) m.mem.Add(float64(stat.Memory.Usage) / 1024 / 1024) m.lastStat = stat - fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) + // fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) } func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 97a890a6ae..ceab6e5cf0 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -4,50 +4,48 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "time" - "github.com/alibaba/ilogtail/test/config" "golang.org/x/time/rate" + + "github.com/alibaba/ilogtail/test/config" ) func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, templateStr string) (context.Context, error) { - // 打开或创建文件 - if !filepath.IsAbs(path) { - path = filepath.Join(config.CaseHome, path) - } - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return ctx, err - } + // clear file + path = filepath.Join(config.CaseHome, path) + path = filepath.Clean(path) + _ = os.WriteFile(path, []byte{}, 0600) + + command := fmt.Sprintf("echo '%s' >> %s", templateStr, path) // interval = templateLen / speed interval := time.Microsecond * time.Duration(len(templateStr)/speed) limiter := rate.NewLimiter(rate.Every(interval), 1) - // 总时间控制 timeout := time.After(time.Minute * time.Duration(totalTime)) for { select { - case <-ctx.Done(): // 上下文取消 - if err := file.Close(); err != nil { - return ctx, err - } + // context is done + case <-ctx.Done(): + // clear file + _ = os.WriteFile(path, []byte{}, 0600) return ctx, ctx.Err() - case <-timeout: // 总时间到 - if err := file.Close(); err != nil { - return ctx, err - } + // all time is done + case <-timeout: + // clear file + _ = os.WriteFile(path, []byte{}, 0600) return ctx, nil default: if err := limiter.Wait(ctx); err != nil { return ctx, err } - if _, err := fmt.Fprintln(file, templateStr); err != nil { - return ctx, err - } + cmd := exec.Command("sh", "-c", command) + _ = cmd.Run() } } } From 9d700f557e37b8d74e2f40ef6a6ff8e474ffb50a Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Sat, 20 Jul 2024 20:16:52 +0800 Subject: [PATCH 04/88] add benchmark CI --- .github/workflows/benchmark.yaml | 98 +++++++++++++++++++ Makefile | 9 +- scripts/benchmark_collect_result.sh | 24 +++++ .../docker-compose.yaml | 22 ----- .../filebeat.yml | 15 +-- .../main.conf | 16 ++- .../parsers.conf | 2 +- .../docker-compose.yaml | 3 +- .../vector.yaml | 23 ++++- test/engine/setup/monitor/monitor.go | 31 ++---- test/engine/setup/monitor/statistic.go | 32 +++++- 11 files changed, 208 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/benchmark.yaml create mode 100755 scripts/benchmark_collect_result.sh diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 0000000000..14faf4801c --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,98 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Benchmark + +on: + pull_request: + paths-ignore: + - 'docs/**' + - 'example_config/**' + - 'docker/**' + - 'k8s_template/**' + - 'changes/**' + - 'licenses/**' + - 'CHANGELOG.md' + push: + branches: + - main + - 1.* +jobs: + CI: + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + matrix: + go-version: [ 1.19.10 ] + runner: [ ubuntu-latest ] + fail-fast: true + steps: + # Clean up space to prevent action from running out of disk space. + - name: clean + if: matrix.runner == 'ubuntu-latest' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo -E apt-get -qq autoremove --purge + sudo -E apt-get -qq clean + + - name: Check disk space + run: | + df -hT $PWD + + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Check out code + uses: actions/checkout@v2 + with: + submodules: true + + - name: Update Docker-compose to v2 + run: | + sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + + - name: System environment + run: | + uname -r + docker --version + go version + + - name: Run benchmark + env: + BUILD_LOGTAIL_UT: OFF + WITHOUTGDB: ON + run: make benchmark + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: benchmark + tool: "customSmallerIsBetter" + output-file-path: "test/benchmark/report/combined_benchmark.json" + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + result: + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: [ CI ] + steps: + - name: Build Result + run: echo "Just to make the GitHub merge button green" \ No newline at end of file diff --git a/Makefile b/Makefile index dea4e8a51d..da1bea47c8 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ e2e-performance: clean docker gocdocker .PHONY: unittest_e2e_engine unittest_e2e_engine: clean gocdocker - cd test && go test $$(go list ./... | grep -Ev "engine|e2e") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready + cd test && go test $$(go list ./... | grep -Ev "engine|e2e|benchmark") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready .PHONY: unittest_plugin unittest_plugin: clean import_plugins @@ -209,6 +209,13 @@ unittest_pluginmanager: clean import_plugins go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go +# benchmark +.PHONY: benchmark +benchmark: clean gocdocker e2edocker + ./scripts/e2e.sh benchmark performance + ./scripts/benchmark_collect_result.sh + + .PHONY: all all: clean import_plugins ./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE) diff --git a/scripts/benchmark_collect_result.sh b/scripts/benchmark_collect_result.sh new file mode 100755 index 0000000000..1eeac303c7 --- /dev/null +++ b/scripts/benchmark_collect_result.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Define the input files and the output file +input_files=($(find test/benchmark/report -type f -name '*benchmark.json')) +output_file="test/benchmark/report/combined_benchmark.json" + +# Start the output file with an opening square bracket +rm -f "$output_file" +touch "$output_file" +echo '[' > "$output_file" + +# Loop through each input file +for i in "${!input_files[@]}"; do + # Read the file, remove the first and last line (the square brackets), and append to the output file + cat "${input_files[$i]}" | sed '1d;$d' >> "$output_file" + + # If this is not the last file, append a comma to separate the arrays + if [ $i -lt $((${#input_files[@]} - 1)) ]; then + echo ',' >> "$output_file" + fi +done + +# Finish the output file with a closing square bracket +echo ']' >> "$output_file" \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml index 8e12d4a340..9e90378fa9 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -1,22 +1,6 @@ version: '3.8' services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.2 - environment: - - ELASTIC_PASSWORD=elastic - - discovery.type=single-node - - xpack.security.http.ssl.enabled=false - - xpack.license.self_generated.type=trial - ports: - - 9200:9200 - healthcheck: - test: ["CMD-SHELL", "curl -u elastic:elastic -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] - interval: 10s - timeout: 5s - retries: 3 - restart: always - filebeat: image: docker.elastic.co/beats/filebeat:8.14.2 user: root @@ -24,9 +8,3 @@ services: - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - ./a.log:/home/filebeat/a.log:ro command: filebeat -e --strict.perms=false - environment: - - OUTPUT_ELASTICSEARCH_HOSTS=["elasticsearch:9200"] - depends_on: - elasticsearch: - condition: service_healthy - restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml index 1c95c5ec77..e2de4b2806 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -7,11 +7,12 @@ filebeat.inputs: processors: - decode_json_fields: fields: ["message"] - process_array: false - max_depth: 1 - target: "" - overwrite_keys: false - add_error_key: true + target: "json" + - drop_event: + when: + not: + equals: + json.user-agent: "no-agent" -output.discard: - enabled: true +output.console: + pretty: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf index baaf715b82..7436cae6b6 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -1,10 +1,16 @@ [SERVICE] - Parsers+File /tmp/parsers.conf + parsers_file /tmp/parsers.conf [INPUT] - Name tail - Path /home/fluentbit/a.log + name tail + path /home/fluentbit/a.log + parser json + +[FILTER] + name grep + match * + regex user-agent no-agent [OUTPUT] - Name stdout - Match non + name stdout + match * diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf index 27cc2efc00..9229d0ab6a 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf @@ -1,3 +1,3 @@ [PARSER] - Name docker + Name json Format json \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml index 3763d9a3e6..c133cb69ec 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -5,5 +5,4 @@ services: image: timberio/vector:0.39.0-debian volumes: - ./vector.yaml:/etc/vector/vector.yaml - - ./a.log:/home/vector-log/a.log - restart: always \ No newline at end of file + - ./a.log:/home/vector-log/a.log \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml index cd13fc3489..c108a3bdc8 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml @@ -4,8 +4,25 @@ sources: include: - /home/vector-log/*.log +transforms: + transform_json: + type: remap + inputs: + - input_file + source: |- + . = parse_json!(string!(.message)) + filter_agent: + type: filter + inputs: + - transform_json + condition: + type: "vrl" + source: ."usr-agent" == "no-agent" + sinks: - my_sink_id: - type: blackhole + output_std: + type: console inputs: - - input_file \ No newline at end of file + - filter_agent + encoding: + codec: json \ No newline at end of file diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 3d65521df6..214c47ddf2 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -3,10 +3,8 @@ package monitor import ( "context" "fmt" - "log" "os" "path/filepath" - "strings" "sync/atomic" "time" @@ -53,38 +51,21 @@ func StopMonitor(ctx context.Context) (context.Context, error) { func monitoring(client *client.Client, containerName string) { // create csv file - reportDir := filepath.Join(config.CaseHome, "report") - reportDir, err := filepath.Abs(reportDir) - if err != nil { - log.Fatalf("Failed to get absolute path: %s", err) - } - if _, err = os.Stat(reportDir); os.IsNotExist(err) { - // 文件夹不存在，创建文件夹 - err = os.MkdirAll(reportDir, 0750) - if err != nil { - log.Fatalf("Failed to create folder: %s", err) - } - } + root, _ := filepath.Abs(".") + reportDir := root + "/report/" + benchmarkFile := reportDir + config.CaseName + "_benchmark.json" // new ticker ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() // read from cadvisor per interval seconds request := &v1.ContainerInfoRequest{NumStats: 10} - monitorStatistic := NewMonitorStatistic() + monitorStatistic := NewMonitorStatistic(config.CaseName) for { select { case <-stopCh: isMonitoring.Store(false) - var builder strings.Builder - builder.WriteString("Metric,Value\n") - builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Max(%)", monitorStatistic.cpu.maxVal)) - builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Avg(%)", monitorStatistic.cpu.avgVal)) - builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Max(MB)", monitorStatistic.mem.maxVal)) - builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Avg(MB)", monitorStatistic.mem.avgVal)) - err = os.WriteFile(filepath.Join(reportDir, "monitor.csv"), []byte(builder.String()), 0600) - if err != nil { - log.Default().Printf("Failed to write monitor result: %s", err) - } + bytes, _ := monitorStatistic.MarshalJSON() + _ = os.WriteFile(benchmarkFile, bytes, 0600) return case <-ticker.C: // 获取容器信息 diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index 442628dfe6..efa5eb54ba 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -1,6 +1,9 @@ package monitor import ( + "encoding/json" + "fmt" + v1 "github.com/google/cadvisor/info/v1" ) @@ -25,13 +28,15 @@ func (s *Info) Add(val float64) { } type Statistic struct { + name string cpu Info mem Info lastStat *v1.ContainerStats } -func NewMonitorStatistic() *Statistic { +func NewMonitorStatistic(name string) *Statistic { return &Statistic{ + name: name, cpu: Info{}, mem: Info{}, lastStat: nil, @@ -57,3 +62,28 @@ func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { cpuUsageRateTotal := float64(cpuUsageTotal) * 100 / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds()) return cpuUsageRateTotal } + +type StatisticItem struct { + Name string `json:"name"` + Value float64 `json:"value"` + Unit string `json:"unit"` +} + +func (m *Statistic) MarshalJSON() ([]byte, error) { + items := []StatisticItem{ + {"CPU_Usage_Max-" + m.name, m.cpu.maxVal, "%"}, + {"CPU_Usage_Avg-" + m.name, m.cpu.avgVal, "%"}, + {"Memory_Usage_Max-" + m.name, m.mem.maxVal, "MB"}, + {"Memory_Usage_Avg-" + m.name, m.mem.avgVal, "MB"}, + } + + // Serialize the slice to JSON + jsonData, err := json.MarshalIndent(items, "", " ") + if err != nil { + fmt.Println("Error serializing statistics:", err) + return nil, err + } + + // Output the JSON string + return jsonData, nil +} From 49812f930c14b58135fb65a95fecae2ca968ae92 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Sun, 21 Jul 2024 16:39:39 +0800 Subject: [PATCH 05/88] update benmark.yaml from using benchmark action to just cat result.json --- .github/workflows/benchmark.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 14faf4801c..d1e9af0041 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -80,14 +80,8 @@ jobs: WITHOUTGDB: ON run: make benchmark - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: benchmark - tool: "customSmallerIsBetter" - output-file-path: "test/benchmark/report/combined_benchmark.json" - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + - name: Show benchmark result + run: cat test/benchmark/report/combined_benchmark.json result: runs-on: ubuntu-latest From 2a455f58212a3be212b7b563becc5679692a6a99 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Tue, 23 Jul 2024 16:42:00 +0800 Subject: [PATCH 06/88] add CI summary --- .github/workflows/benchmark.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index d1e9af0041..b4ad011902 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -80,8 +80,21 @@ jobs: WITHOUTGDB: ON run: make benchmark - - name: Show benchmark result - run: cat test/benchmark/report/combined_benchmark.json + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: benchmark + tool: "customSmallerIsBetter" + output-file-path: "test/benchmark/report/combined_benchmark.json" + github-token: ${{ secrets.GITHUB_TOKEN }} + external-data-json-path: ./cache/benchmark-data.json + summary-always: true result: runs-on: ubuntu-latest From 11e7aa3379633cf8e3c6c2e0138a34e017673783 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Wed, 24 Jul 2024 10:55:05 +0800 Subject: [PATCH 07/88] trigger benchmark CI From c545ad962745d3fd08115eaee0742a0e59fdb1f8 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Wed, 24 Jul 2024 11:47:01 +0800 Subject: [PATCH 08/88] modify benmark.yaml add summary --- .github/workflows/benchmark.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index b4ad011902..93cff57cac 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -92,7 +92,6 @@ jobs: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/combined_benchmark.json" - github-token: ${{ secrets.GITHUB_TOKEN }} external-data-json-path: ./cache/benchmark-data.json summary-always: true From 3f3521a52c53599531ae21c66393d03914b1dfde Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Thu, 25 Jul 2024 15:25:35 +0800 Subject: [PATCH 09/88] update benchmark CI to only trigger on merge, update test_cases case.feature to monitor on service name --- .github/workflows/benchmark.yaml | 11 ++++--- test/benchmark/benchmark_docker_test.go | 32 ------------------- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- test/engine/setup/monitor/monitor.go | 25 +++++++++------ 7 files changed, 27 insertions(+), 49 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 93cff57cac..5ba13e747d 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -16,6 +16,8 @@ name: Benchmark on: pull_request: + branches: + - main paths-ignore: - 'docs/**' - 'example_config/**' @@ -24,12 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - push: - branches: - - main - - 1.* + types: + - closed + jobs: CI: + if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -96,6 +98,7 @@ jobs: summary-always: true result: + if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] diff --git a/test/benchmark/benchmark_docker_test.go b/test/benchmark/benchmark_docker_test.go index 9a560bce68..493d0567c2 100644 --- a/test/benchmark/benchmark_docker_test.go +++ b/test/benchmark/benchmark_docker_test.go @@ -19,38 +19,6 @@ import ( "github.com/cucumber/godog" ) -func TestE2EOnDockerCompose(t *testing.T) { - suite := godog.TestSuite{ - Name: "E2EOnDockerCompose", - ScenarioInitializer: scenarioInitializer, - Options: &godog.Options{ - Format: "pretty", - Paths: []string{"test_cases"}, - Tags: "@e2e && @docker-compose && ~@ebpf", - TestingT: t, - }, - } - if suite.Run() != 0 { - t.Fail() - } -} - -func TestE2EOnDockerComposeCore(t *testing.T) { - suite := godog.TestSuite{ - Name: "E2EOnDockerCompose", - ScenarioInitializer: scenarioInitializer, - Options: &godog.Options{ - Format: "pretty", - Paths: []string{"test_cases"}, - Tags: "@e2e-core && @docker-compose && ~@ebpf", - TestingT: t, - }, - } - if suite.Run() != 0 { - t.Fail() - } -} - func TestE2EOnDockerComposePerformance(t *testing.T) { suite := godog.TestSuite{ Name: "E2EOnDockerCompose", diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 5285082080..951b42cbe8 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -7,7 +7,7 @@ Feature: performance file to blackhole filebeat Given {docker-compose} environment Given docker-compose type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} - When start monitor {e2e-filebeat-1} + When start monitor {filebeat} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index a29fac66f8..712f28aa09 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -7,7 +7,7 @@ Feature: performance file to blackhole fluentbit Given {docker-compose} environment Given docker-compose type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} - When start monitor {e2e-fluent-bit-1} + When start monitor {fluent-bit} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index eceb6534fc..f541582d5a 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -30,7 +30,7 @@ Feature: performance file to blackhole iLogtail """ Given iLogtail container mount {./a.log} to {/home/test-log/json.log} When start docker-compose {performance_file_to_blackhole_ilogtail} - When start monitor {e2e-ilogtailC-1} + When start monitor {ilogtailC} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index b6265805d4..e76cf6bc23 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -7,7 +7,7 @@ Feature: performance file to blackhole vector Given {docker-compose} environment Given docker-compose type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} - When start monitor {e2e-vector-1} + When start monitor {vector} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 214c47ddf2..518c1aa711 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync/atomic" "time" @@ -28,18 +29,24 @@ func StartMonitor(ctx context.Context, containerName string) (context.Context, e if err != nil { return ctx, err } - // 获取机器信息 - request := &v1.ContainerInfoRequest{NumStats: 10} - _, err = client.DockerContainer(containerName, request) + // 获取所有容器信息 + allContainers, err := client.AllDockerContainers(&v1.ContainerInfoRequest{NumStats: 10}) if err != nil { - fmt.Println("Error getting container info:", err) + fmt.Println("Error getting all containers info:", err) return ctx, err } - stopCh = make(chan bool) - isMonitoring.Store(true) - fmt.Println("Start monitoring container:", containerName) - go monitoring(client, containerName) - return ctx, nil + for _, container := range allContainers { + containerFullName := container.Aliases[0] + if strings.Contains(containerFullName, containerName) { + stopCh = make(chan bool) + isMonitoring.Store(true) + fmt.Println("Start monitoring container:", containerFullName) + go monitoring(client, containerFullName) + return ctx, nil + } + } + err = fmt.Errorf("container %s not found", containerName) + return ctx, err } func StopMonitor(ctx context.Context) (context.Context, error) { From 9dfd2db354f30a89b986265c1b0f30a2e048bc6b Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Mon, 29 Jul 2024 11:09:00 +0800 Subject: [PATCH 10/88] add benchmark docs benchmark.md and delete a.log --- docs/cn/developer-guide/test/benchmark.md | 60 +++++++++++++++++++ .../a.log | 0 .../docker-compose.yaml | 2 +- .../filebeat.yml | 3 +- .../a.log | 0 .../docker-compose.yaml | 2 +- .../main.conf | 3 +- .../a.log | 0 .../case.feature | 14 ++--- .../a.log | 0 .../docker-compose.yaml | 2 +- .../setup/dockercompose/compose benchmark.go | 9 +-- 12 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 docs/cn/developer-guide/test/benchmark.md delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md new file mode 100644 index 0000000000..ede95de9d6 --- /dev/null +++ b/docs/cn/developer-guide/test/benchmark.md @@ -0,0 +1,60 @@ +# Benchmark测试 + +## 工作原理 +与e2e测试工作原理相同，详情可见： +- [e2e-test.md](./e2e-test.md) + +## 测试流程 + +### 环境准备 + +目前仅支持通过docker-compose搭建测试环境，在准备开始进行benchmark测试前，您首先需要准备以下内容： + +- 测试环境：Docker-Compose环境（需在本地安装docker-compose） + +### 配置文件 + +对于每一个新的benchmark，您需要在`./test/benchmark/test_cases/`目录下创建一个新的feature配置文件。每个配置文件中可以包含多个测试场景，每个测试场景由一个或多个步骤组成。 + +配置文件的基本框架如下： + +```plain +@input +Feature: performance file to blackhole vector + Performance file to blackhole vector + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeVector + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_vector} + When start monitor {vector} + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ +``` + +- `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 +- `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 +- `Given docker-compose type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 +- `When start docker-compose {scenario directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`scenario director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 +- `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 +- `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) + +### 运行测试 + +在所有测试内容准备完毕后，您可以直接在test目录下以go test的方式运行E2E测试。根据您的需求，可以选择运行所有测试或者指定测试。 + +```shell +# 运行所有benchmark测试 +go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$ github.com/alibaba/ilogtail/test/benchmark + +# 以正则表达式匹配Scenario运行测试，以下为运行Scenario以Vector为结尾的test_case +go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ github.com/alibaba/ilogtail/test/benchmark +``` + +### 测试结果 + +- 所有测试结果将以json格式记录在`test/benchmark/report/.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数。 +- 运行`scripts/benchmark_collect_result.sh`会将`test/benchmark/report/`下所有结果收集并生成汇总结果到`test/benchmark/report/combines_result.json`，主要用于github benmark action收集本次测试数据并与历史数据汇总 \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml index 9e90378fa9..ce6b83c442 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -6,5 +6,5 @@ services: user: root volumes: - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - ./a.log:/home/filebeat/a.log:ro + - .:/home/filebeat command: filebeat -e --strict.perms=false diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml index e2de4b2806..e93a2688e0 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -2,7 +2,8 @@ filebeat.inputs: - type: filestream id: input-file paths: - - /home/filebeat/a.log + - /home/filebeat/*.log + prospector.scanner.check_interval: 1s processors: - decode_json_fields: diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml index aace706b7b..aeb15b1f07 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml @@ -8,7 +8,7 @@ services: volumes: - ./main.conf:/tmp/main.conf - ./parsers.conf:/tmp/parsers.conf - - ./a.log:/home/fluentbit/a.log + - .:/home/fluentbit restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf index 7436cae6b6..d009876b60 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -3,8 +3,9 @@ [INPUT] name tail - path /home/fluentbit/a.log + path /home/fluentbit/*.log parser json + refresh_interval 1 [FILTER] name grep diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index f541582d5a..8c416bd168 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -12,7 +12,7 @@ Feature: performance file to blackhole iLogtail inputs: - Type: input_file FilePaths: - - /home/test-log/json.log + - /home/test-log/*.log processors: - Type: processor_parse_json_native SourceKey: content @@ -22,13 +22,13 @@ Feature: performance file to blackhole iLogtail FilterRegex: - ^no-agent$ flushers: - - Type: flusher_sls - Region: cn-hangzhou - Endpoint: cn-hangzhou.log.aliyuncs.com - Project: test_project - Logstore: test_logstore + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore """ - Given iLogtail container mount {./a.log} to {/home/test-log/json.log} + Given iLogtail container mount {.} to {/home/test-log} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml index c133cb69ec..9c35dcd6d3 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -5,4 +5,4 @@ services: image: timberio/vector:0.39.0-debian volumes: - ./vector.yaml:/etc/vector/vector.yaml - - ./a.log:/home/vector-log/a.log \ No newline at end of file + - .:/home/vector-log \ No newline at end of file diff --git a/test/engine/setup/dockercompose/compose benchmark.go b/test/engine/setup/dockercompose/compose benchmark.go index 4b566c10a3..f82607db4c 100644 --- a/test/engine/setup/dockercompose/compose benchmark.go +++ b/test/engine/setup/dockercompose/compose benchmark.go @@ -30,7 +30,8 @@ import ( ) const ( - cadvisorTemplate = `version: '3.8' + benchmarkIdentifier = "benchmark" + cadvisorTemplate = `version: '3.8' services: cadvisor: image: gcr.io/cadvisor/cadvisor:v0.49.1 @@ -64,7 +65,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { if err := c.createComposeFile(); err != nil { return err } - compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).WithCommand([]string{"up", "-d", "--build"}) + compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).WithCommand([]string{"up", "-d", "--build"}) strategyWrappers := withExposedService(compose) execError := compose.Invoke() if execError.Error != nil { @@ -79,7 +80,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { c.cli = cli list, err := cli.ContainerList(context.Background(), types.ContainerListOptions{ - Filters: filters.NewArgs(filters.Arg("name", "e2e-cadvisor")), + Filters: filters.NewArgs(filters.Arg("name", "benchmark-cadvisor")), }) if len(list) != 1 { logger.Errorf(context.Background(), "CADVISOR_COMPOSE_ALARM", "cadvisor container size is not equal 1, got %d count", len(list)) @@ -102,7 +103,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).Down() + execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", "stdout", execError.Stdout.Error(), "stderr", execError.Stderr.Error()) From 439fbe1959bc0a2a195ded2c1b63f8c797a187d0 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Wed, 7 Aug 2024 20:38:25 +0800 Subject: [PATCH 11/88] update github benchmark CI to store avg/max ans time records --- .github/workflows/benchmark.yaml | 25 +++++++++++ scripts/benchmark_collect_result.sh | 45 ++++++++++++++----- .../docker-compose.yaml | 2 +- test/engine/setup/monitor/monitor.go | 9 ++-- test/engine/setup/monitor/statistic.go | 35 ++++++++++++--- 5 files changed, 97 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 5ba13e747d..f3a60e1dd0 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -88,14 +88,39 @@ jobs: path: ./cache key: ${{ runner.os }}-benchmark +<<<<<<< HEAD - name: Store benchmark result +======= + - name: Store benchmark result iLogtail +>>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" +<<<<<<< HEAD output-file-path: "test/benchmark/report/combined_benchmark.json" external-data-json-path: ./cache/benchmark-data.json summary-always: true +======= + output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" + external-data-json-path: ./cache/ilogtail_statistic.json + auto-push: false + summary-always: true + + # - name: Store benchmark result other agents + # run: | + # git checkout gh-pages + # cp test/benchmark/report/records_all.json dev/bench/records_all.js + # sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js + # git add dev/bench/records_all.js + # last_commit_message=$(git log -1 --pretty=%B) + # git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" + + # # Push gh-pages branch by yourself + # - name: Push benchmark result + # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + +>>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) result: if: github.event.pull_request.merged == true diff --git a/scripts/benchmark_collect_result.sh b/scripts/benchmark_collect_result.sh index 1eeac303c7..27a1625eb0 100755 --- a/scripts/benchmark_collect_result.sh +++ b/scripts/benchmark_collect_result.sh @@ -1,24 +1,49 @@ #!/bin/bash +# Combine the statistic and records files from each benchmark run into a single file + +# Define the input files and the output file +input_statistic_files=($(find test/benchmark/report -type f -name '*ilogtail_statistic.json')) +output_statistic_file="test/benchmark/report/ilogtail_statistic_all.json" + +# Start the output file with an opening square bracket +rm -f "$output_statistic_file" +touch "$output_statistic_file" +echo '[' > "$output_statistic_file" + +# Loop through each input file +for i in "${!input_statistic_files[@]}"; do + # Read the file, remove the first and last line (the square brackets), and append to the output file + cat "${input_statistic_files[$i]}" | sed '1d;$d' >> "$output_statistic_file" + + # If this is not the last file, append a comma to separate the arrays + if [ $i -lt $((${#input_statistic_files[@]} - 1)) ]; then + echo ',' >> "$output_statistic_file" + fi +done + +# Finish the output file with a closing square bracket +echo ']' >> "$output_statistic_file" + # Define the input files and the output file -input_files=($(find test/benchmark/report -type f -name '*benchmark.json')) -output_file="test/benchmark/report/combined_benchmark.json" +input_records_files=($(find test/benchmark/report -type f -name '*records.json')) +output_records_file="test/benchmark/report/records_all.json" # Start the output file with an opening square bracket -rm -f "$output_file" -touch "$output_file" -echo '[' > "$output_file" +rm -f "$output_records_file" +touch "$output_records_file" +echo '[' > "$output_records_file" # Loop through each input file -for i in "${!input_files[@]}"; do +for i in "${!input_records_files[@]}"; do # Read the file, remove the first and last line (the square brackets), and append to the output file - cat "${input_files[$i]}" | sed '1d;$d' >> "$output_file" + cat "${input_records_files[$i]}" | sed '1d;$d' >> "$output_records_file" # If this is not the last file, append a comma to separate the arrays - if [ $i -lt $((${#input_files[@]} - 1)) ]; then - echo ',' >> "$output_file" + if [ $i -lt $((${#input_records_files[@]} - 1)) ]; then + echo ',' >> "$output_records_file" fi done # Finish the output file with a closing square bracket -echo ']' >> "$output_file" \ No newline at end of file +echo ']' >> "$output_records_file" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml index 9c35dcd6d3..635282b58d 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -5,4 +5,4 @@ services: image: timberio/vector:0.39.0-debian volumes: - ./vector.yaml:/etc/vector/vector.yaml - - .:/home/vector-log \ No newline at end of file + - .:/home/vector-log diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 518c1aa711..c8898d95ca 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -60,7 +60,8 @@ func monitoring(client *client.Client, containerName string) { // create csv file root, _ := filepath.Abs(".") reportDir := root + "/report/" - benchmarkFile := reportDir + config.CaseName + "_benchmark.json" + statisticFile := reportDir + config.CaseName + "_statistic.json" + recordsFile := reportDir + config.CaseName + "_records.json" // new ticker ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() @@ -71,8 +72,10 @@ func monitoring(client *client.Client, containerName string) { select { case <-stopCh: isMonitoring.Store(false) - bytes, _ := monitorStatistic.MarshalJSON() - _ = os.WriteFile(benchmarkFile, bytes, 0600) + bytes, _ := monitorStatistic.MarshalStatisticJSON() + _ = os.WriteFile(statisticFile, bytes, 0600) + bytes, _ = monitorStatistic.MarshalRecordsJSON() + _ = os.WriteFile(recordsFile, bytes, 0600) return case <-ticker.C: // 获取容器信息 diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index efa5eb54ba..83ea2934b7 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -11,9 +11,11 @@ type Info struct { maxVal float64 avgVal float64 cnt int64 + values []float64 } func (s *Info) Add(val float64) { + s.values = append(s.values, val) if s.cnt < 1 { s.maxVal = val s.avgVal = val @@ -69,12 +71,35 @@ type StatisticItem struct { Unit string `json:"unit"` } -func (m *Statistic) MarshalJSON() ([]byte, error) { +func (m *Statistic) MarshalStatisticJSON() ([]byte, error) { items := []StatisticItem{ - {"CPU_Usage_Max-" + m.name, m.cpu.maxVal, "%"}, - {"CPU_Usage_Avg-" + m.name, m.cpu.avgVal, "%"}, - {"Memory_Usage_Max-" + m.name, m.mem.maxVal, "MB"}, - {"Memory_Usage_Avg-" + m.name, m.mem.avgVal, "MB"}, + {"CPU_Usage_Max - " + m.name, m.cpu.maxVal, "%"}, + {"CPU_Usage_Avg - " + m.name, m.cpu.avgVal, "%"}, + {"Memory_Usage_Max - " + m.name, m.mem.maxVal, "MB"}, + {"Memory_Usage_Avg - " + m.name, m.mem.avgVal, "MB"}, + } + + // Serialize the slice to JSON + jsonData, err := json.MarshalIndent(items, "", " ") + if err != nil { + fmt.Println("Error serializing statistics:", err) + return nil, err + } + + // Output the JSON string + return jsonData, nil +} + +type RecordsItem struct { + Name string `json:"name"` + Records []float64 `json:"records"` + Unit string `json:"unit"` +} + +func (m *Statistic) MarshalRecordsJSON() ([]byte, error) { + items := []RecordsItem{ + {"CPU_Usage - " + m.name, m.cpu.values, "%"}, + {"Memory_Usage - " + m.name, m.mem.values, "MB"}, } // Serialize the slice to JSON From d4562e0e5b0872c8fb6b36622c1531c157e98328 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Sun, 11 Aug 2024 06:43:21 +0000 Subject: [PATCH 12/88] update trigger file.go --- .github/workflows/benchmark.yaml | 18 +++++------------- .github/workflows/static-check.yaml | 1 + test/engine/trigger/file.go | 20 ++++++-------------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index f3a60e1dd0..40f9db5ba8 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -28,10 +28,13 @@ on: - 'CHANGELOG.md' types: - closed + push: + branches: + - benchmark jobs: CI: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -88,20 +91,11 @@ jobs: path: ./cache key: ${{ runner.os }}-benchmark -<<<<<<< HEAD - - name: Store benchmark result -======= - name: Store benchmark result iLogtail ->>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" -<<<<<<< HEAD - output-file-path: "test/benchmark/report/combined_benchmark.json" - external-data-json-path: ./cache/benchmark-data.json - summary-always: true -======= output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false @@ -120,8 +114,6 @@ jobs: # - name: Push benchmark result # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages ->>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) - result: if: github.event.pull_request.merged == true runs-on: ubuntu-latest @@ -129,4 +121,4 @@ jobs: needs: [ CI ] steps: - name: Build Result - run: echo "Just to make the GitHub merge button green" \ No newline at end of file + run: echo "Just to make the GitHub merge button green" diff --git a/.github/workflows/static-check.yaml b/.github/workflows/static-check.yaml index cbeaa8c8d4..ccaa148f3f 100644 --- a/.github/workflows/static-check.yaml +++ b/.github/workflows/static-check.yaml @@ -28,6 +28,7 @@ on: branches: - main - 1.* + - benchmark jobs: CI: env: diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index ceab6e5cf0..5cf00a2bbd 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -2,9 +2,7 @@ package trigger import ( "context" - "fmt" "os" - "os/exec" "path/filepath" "time" @@ -18,13 +16,9 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t path = filepath.Join(config.CaseHome, path) path = filepath.Clean(path) _ = os.WriteFile(path, []byte{}, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - command := fmt.Sprintf("echo '%s' >> %s", templateStr, path) - - // interval = templateLen / speed - interval := time.Microsecond * time.Duration(len(templateStr)/speed) - - limiter := rate.NewLimiter(rate.Every(interval), 1) + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), len(templateStr)) timeout := time.After(time.Minute * time.Duration(totalTime)) @@ -33,19 +27,17 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t // context is done case <-ctx.Done(): // clear file - _ = os.WriteFile(path, []byte{}, 0600) + _ = file.Close() return ctx, ctx.Err() // all time is done case <-timeout: // clear file - _ = os.WriteFile(path, []byte{}, 0600) + _ = file.Close() return ctx, nil default: - if err := limiter.Wait(ctx); err != nil { - return ctx, err + if limiter.AllowN(time.Now(), len(templateStr)) { + _, _ = file.WriteString(templateStr + "\n") } - cmd := exec.Command("sh", "-c", command) - _ = cmd.Run() } } } From 13204b0fde31b6a489cfaa465f9d3623fee128f5 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Sun, 11 Aug 2024 15:21:43 +0800 Subject: [PATCH 13/88] update trigger file.go lint error --- test/engine/trigger/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 5cf00a2bbd..aaa47deb3f 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -16,7 +16,7 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t path = filepath.Join(config.CaseHome, path) path = filepath.Clean(path) _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), len(templateStr)) From 8e2a2688dc71d903f59dec2aec964b1e5fcd59c1 Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Sun, 11 Aug 2024 15:54:22 +0800 Subject: [PATCH 14/88] update trigger file.go --- .github/workflows/benchmark.yaml | 13 +------------ test/engine/trigger/file.go | 20 ++++++-------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index f3a60e1dd0..b63b35c83e 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -88,20 +88,11 @@ jobs: path: ./cache key: ${{ runner.os }}-benchmark -<<<<<<< HEAD - - name: Store benchmark result -======= - name: Store benchmark result iLogtail ->>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" -<<<<<<< HEAD - output-file-path: "test/benchmark/report/combined_benchmark.json" - external-data-json-path: ./cache/benchmark-data.json - summary-always: true -======= output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false @@ -120,8 +111,6 @@ jobs: # - name: Push benchmark result # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages ->>>>>>> a9705b46 (update github benchmark CI to store avg/max ans time records) - result: if: github.event.pull_request.merged == true runs-on: ubuntu-latest @@ -129,4 +118,4 @@ jobs: needs: [ CI ] steps: - name: Build Result - run: echo "Just to make the GitHub merge button green" \ No newline at end of file + run: echo "Just to make the GitHub merge button green" diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index ceab6e5cf0..aaa47deb3f 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -2,9 +2,7 @@ package trigger import ( "context" - "fmt" "os" - "os/exec" "path/filepath" "time" @@ -18,13 +16,9 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t path = filepath.Join(config.CaseHome, path) path = filepath.Clean(path) _ = os.WriteFile(path, []byte{}, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - command := fmt.Sprintf("echo '%s' >> %s", templateStr, path) - - // interval = templateLen / speed - interval := time.Microsecond * time.Duration(len(templateStr)/speed) - - limiter := rate.NewLimiter(rate.Every(interval), 1) + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), len(templateStr)) timeout := time.After(time.Minute * time.Duration(totalTime)) @@ -33,19 +27,17 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t // context is done case <-ctx.Done(): // clear file - _ = os.WriteFile(path, []byte{}, 0600) + _ = file.Close() return ctx, ctx.Err() // all time is done case <-timeout: // clear file - _ = os.WriteFile(path, []byte{}, 0600) + _ = file.Close() return ctx, nil default: - if err := limiter.Wait(ctx); err != nil { - return ctx, err + if limiter.AllowN(time.Now(), len(templateStr)) { + _, _ = file.WriteString(templateStr + "\n") } - cmd := exec.Command("sh", "-c", command) - _ = cmd.Run() } } } From c1257b0e1914042f4ff462a3f9b4ee364683083e Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Mon, 19 Aug 2024 15:55:38 +0800 Subject: [PATCH 15/88] polish --- docs/cn/developer-guide/test/benchmark.md | 2 +- .../performance_file_to_blackhole_vector/vector.yaml | 2 +- test/engine/setup/dockercompose/common.go | 9 +++++++-- .../{compose benchmark.go => compose_benchmark.go} | 0 4 files changed, 9 insertions(+), 4 deletions(-) rename test/engine/setup/dockercompose/{compose benchmark.go => compose_benchmark.go} (100%) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index ede95de9d6..3715137910 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -38,7 +38,7 @@ Feature: performance file to blackhole vector - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 - `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 - `Given docker-compose type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 -- `When start docker-compose {scenario directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`scenario director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 +- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 - `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml index c108a3bdc8..20bb85b721 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml @@ -17,7 +17,7 @@ transforms: - transform_json condition: type: "vrl" - source: ."usr-agent" == "no-agent" + source: ."user-agent" == "no-agent" sinks: output_std: diff --git a/test/engine/setup/dockercompose/common.go b/test/engine/setup/dockercompose/common.go index 9f35c3b90e..e7d64122e7 100644 --- a/test/engine/setup/dockercompose/common.go +++ b/test/engine/setup/dockercompose/common.go @@ -20,6 +20,11 @@ import ( "sync" ) +const ( + dockerComposeTypeE2E = "e2e" + dockerComposeTypeBenchmark = "benchmark" +) + var networkMapping = make(map[string]string) var mu sync.Mutex var instance Booter @@ -37,9 +42,9 @@ func Load(loadType string) error { mu.Lock() defer mu.Unlock() switch loadType { - case "e2e": + case dockerComposeTypeE2E: instance = NewComposeBooter() - case "benchmark": + case dockerComposeTypeBenchmark: instance = NewComposeBenchmarkBooter() default: return errors.New("invalid load type") diff --git a/test/engine/setup/dockercompose/compose benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go similarity index 100% rename from test/engine/setup/dockercompose/compose benchmark.go rename to test/engine/setup/dockercompose/compose_benchmark.go From b67d5d8ac7c9f3bb4ce14a8ba902769ee2fcf020 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 18 Sep 2024 18:34:12 -0700 Subject: [PATCH 16/88] update benchmark.md --- docs/cn/developer-guide/test/benchmark.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index ede95de9d6..47a0587827 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -56,5 +56,5 @@ go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ gith ### 测试结果 -- 所有测试结果将以json格式记录在`test/benchmark/report/.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数。 -- 运行`scripts/benchmark_collect_result.sh`会将`test/benchmark/report/`下所有结果收集并生成汇总结果到`test/benchmark/report/combines_result.json`，主要用于github benmark action收集本次测试数据并与历史数据汇总 \ No newline at end of file +- 所有统计结果将以json格式记录在`test/benchmark/report/_statistic.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数；所有实时结果序列将以json格式记录在`test/benchmark/report/_records.json`中，目前记录了测试运行过程中的CPU使用率、内存使用率时间序列。 +- 运行`scripts/benchmark_collect_result.sh`会将数据以github benchmark action所需格式汇总，会将`test/benchmark/report/*ilogtail_statistic.json`下所有结果收集并生成汇总结果到`test/benchmark/report/ilogtail_statistic_all.json`中，并将`test/benchmark/report/*records.json`汇总到`test/benchmark/report/records_all.json` \ No newline at end of file From 1760f0afcafd702d260dc36b8a65cba273845076 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 18 Sep 2024 19:23:04 -0700 Subject: [PATCH 17/88] update benchmark.md --- docs/cn/developer-guide/test/benchmark.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index 3715137910..28ea55e758 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -56,5 +56,5 @@ go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ gith ### 测试结果 -- 所有测试结果将以json格式记录在`test/benchmark/report/.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数。 -- 运行`scripts/benchmark_collect_result.sh`会将`test/benchmark/report/`下所有结果收集并生成汇总结果到`test/benchmark/report/combines_result.json`，主要用于github benmark action收集本次测试数据并与历史数据汇总 \ No newline at end of file +- 所有统计结果将以json格式记录在`test/benchmark/report/_statistic.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数；所有实时结果序列将以json格式记录在`test/benchmark/report/_records.json`中，目前记录了测试运行过程中的CPU使用率、内存使用率时间序列。 +- 运行`scripts/benchmark_collect_result.sh`会将数据以github benchmark action所需格式汇总，会将`test/benchmark/report/*ilogtail_statistic.json`下所有结果收集并生成汇总结果到`test/benchmark/report/ilogtail_statistic_all.json`中，并将`test/benchmark/report/*records.json`汇总到`test/benchmark/report/records_all.json` \ No newline at end of file From 05e30c49cc4c6c924141497f70d2e2bcbe560ce5 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 23 Sep 2024 07:09:13 -0700 Subject: [PATCH 18/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 40f9db5ba8..efc97d9341 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -101,18 +101,19 @@ jobs: auto-push: false summary-always: true - # - name: Store benchmark result other agents - # run: | - # git checkout gh-pages - # cp test/benchmark/report/records_all.json dev/bench/records_all.js - # sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js - # git add dev/bench/records_all.js - # last_commit_message=$(git log -1 --pretty=%B) - # git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" + - name: Store benchmark result other agents + run: | + git checkout gh-pages + cp test/benchmark/report/records_all.json dev/bench/records_all.js + sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js + git add dev/bench/records_all.js + last_commit_message=$(git log -1 --pretty=%B) + git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # # Push gh-pages branch by yourself - # - name: Push benchmark result - # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + # Push gh-pages branch by yourself + - name: Push benchmark result + # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages result: if: github.event.pull_request.merged == true From 3d26c8d60b1b6194fedef5d703e1efac2e1affee Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 23 Sep 2024 18:11:34 -0700 Subject: [PATCH 19/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index efc97d9341..d708ba16cd 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -103,7 +103,6 @@ jobs: - name: Store benchmark result other agents run: | - git checkout gh-pages cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From e2cd6199ee993e5f24c88e5dd9a3ab9087b45d66 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 23 Sep 2024 19:04:11 -0700 Subject: [PATCH 20/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index d708ba16cd..efc97d9341 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -103,6 +103,7 @@ jobs: - name: Store benchmark result other agents run: | + git checkout gh-pages cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From c6c86d536295400cba9d125b0cf62325bf484ea5 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 23 Sep 2024 19:04:50 -0700 Subject: [PATCH 21/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index efc97d9341..9972793621 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -103,7 +103,7 @@ jobs: - name: Store benchmark result other agents run: | - git checkout gh-pages + git checkout -b gh-pages cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From 09f1467f5fe590fac76afb931069cb93eb503767 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 23 Sep 2024 22:49:58 -0700 Subject: [PATCH 22/88] update benmark.yam --- .github/workflows/benchmark.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 9972793621..f940790479 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -85,11 +85,11 @@ jobs: WITHOUTGDB: ON run: make benchmark - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark + # - name: Download previous benchmark data + # uses: actions/cache@v4 + # with: + # path: ./cache + # key: ${{ runner.os }}-benchmark - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 @@ -97,13 +97,13 @@ jobs: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" - external-data-json-path: ./cache/ilogtail_statistic.json + # external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false summary-always: true - name: Store benchmark result other agents run: | - git checkout -b gh-pages + git checkout gh-pages cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From 9714ec159f563669c024276cdd3dcfef326eff3f Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Sat, 28 Sep 2024 23:28:13 -0700 Subject: [PATCH 23/88] test --- .github/workflows/benchmark.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index f940790479..6fcee0084e 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -103,7 +103,9 @@ jobs: - name: Store benchmark result other agents run: | + git branch -a git checkout gh-pages + ls cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From 8fb21e7ea754b998d86a196acd1c7369a3eaf55b Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Sun, 29 Sep 2024 00:29:26 -0700 Subject: [PATCH 24/88] polish --- .github/workflows/benchmark.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 6fcee0084e..f940790479 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -103,9 +103,7 @@ jobs: - name: Store benchmark result other agents run: | - git branch -a git checkout gh-pages - ls cp test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js From 49d0a9d88b73f3e305b5ae66c6222b8c147a6606 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 05:07:34 -0700 Subject: [PATCH 25/88] update clean up and benchmark structure --- docs/cn/developer-guide/test/benchmark.md | 2 +- test/benchmark/benchmark_docker_test.go | 4 +- test/benchmark/main_test.go | 124 ------------------ .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- test/engine/cleanup/monitor.go | 11 -- .../setup/controller/docker_compose_boot.go | 4 +- test/engine/setup/docker_compose.go | 17 ++- test/engine/setup/dockercompose/common.go | 14 +- .../setup/dockercompose/compose_benchmark.go | 2 + test/engine/setup/monitor/monitor.go | 4 +- test/engine/steps.go | 4 + 14 files changed, 38 insertions(+), 156 deletions(-) delete mode 100644 test/benchmark/main_test.go delete mode 100644 test/engine/cleanup/monitor.go diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index 28ea55e758..9030f51b01 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -37,7 +37,7 @@ Feature: performance file to blackhole vector - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 - `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 -- `Given docker-compose type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 +- `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 - `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 - `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) diff --git a/test/benchmark/benchmark_docker_test.go b/test/benchmark/benchmark_docker_test.go index 493d0567c2..f029c0831a 100644 --- a/test/benchmark/benchmark_docker_test.go +++ b/test/benchmark/benchmark_docker_test.go @@ -17,12 +17,14 @@ import ( "testing" "github.com/cucumber/godog" + + "github.com/alibaba/ilogtail/test/engine" ) func TestE2EOnDockerComposePerformance(t *testing.T) { suite := godog.TestSuite{ Name: "E2EOnDockerCompose", - ScenarioInitializer: scenarioInitializer, + ScenarioInitializer: engine.ScenarioInitializer, Options: &godog.Options{ Format: "pretty", Paths: []string{"test_cases"}, diff --git a/test/benchmark/main_test.go b/test/benchmark/main_test.go deleted file mode 100644 index fd1e68b501..0000000000 --- a/test/benchmark/main_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package e2e - -import ( - "context" - "os" - "strconv" - "testing" - "time" - - "github.com/cucumber/godog" - - "github.com/alibaba/ilogtail/pkg/logger" - "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/cleanup" - "github.com/alibaba/ilogtail/test/engine/control" - "github.com/alibaba/ilogtail/test/engine/setup" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" - "github.com/alibaba/ilogtail/test/engine/setup/subscriber" - "github.com/alibaba/ilogtail/test/engine/trigger" - "github.com/alibaba/ilogtail/test/engine/verify" -) - -func TestMain(m *testing.M) { - loggerOptions := []logger.ConfigOption{ - logger.OptionAsyncLogger, - } - loggerOptions = append(loggerOptions, logger.OptionInfoLevel) - logger.InitTestLogger(loggerOptions...) - - config.TestConfig = config.Config{} - // Log - config.TestConfig.GeneratedLogDir = os.Getenv("GENERATED_LOG_DIR") - if len(config.TestConfig.GeneratedLogDir) == 0 { - config.TestConfig.GeneratedLogDir = "/tmp/ilogtail" - } - config.TestConfig.WorkDir = os.Getenv("WORK_DIR") - - // SSH - config.TestConfig.SSHUsername = os.Getenv("SSH_USERNAME") - config.TestConfig.SSHIP = os.Getenv("SSH_IP") - config.TestConfig.SSHPassword = os.Getenv("SSH_PASSWORD") - - // K8s - config.TestConfig.KubeConfigPath = os.Getenv("KUBE_CONFIG_PATH") - - // SLS - config.TestConfig.Project = os.Getenv("PROJECT") - config.TestConfig.Logstore = os.Getenv("LOGSTORE") - config.TestConfig.AccessKeyID = os.Getenv("ACCESS_KEY_ID") - config.TestConfig.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET") - config.TestConfig.Endpoint = os.Getenv("ENDPOINT") - config.TestConfig.Aliuid = os.Getenv("ALIUID") - config.TestConfig.QueryEndpoint = os.Getenv("QUERY_ENDPOINT") - config.TestConfig.Region = os.Getenv("REGION") - timeout, err := strconv.ParseInt(os.Getenv("RETRY_TIMEOUT"), 10, 64) - if err != nil { - timeout = 60 - } - config.TestConfig.RetryTimeout = time.Duration(timeout) * time.Second - code := m.Run() - logger.Flush() - os.Exit(code) -} - -func scenarioInitializer(ctx *godog.ScenarioContext) { - // Given - ctx.Given(`^\{(\S+)\} environment$`, setup.InitEnv) - ctx.Given(`^docker-compose type \{(\S+)\}$`, setup.SetDockerComposeType) - ctx.Given(`^iLogtail depends on containers \{(.*)\}`, setup.SetDockerComposeDependOn) - ctx.Given(`^iLogtail container mount \{(.*)\} to \{(.*)\}`, setup.MountVolume) - ctx.Given(`^iLogtail expose port \{(.*)\} to \{(.*)\}`, setup.ExposePort) - ctx.Given(`^\{(.*)\} local config as below`, control.AddLocalConfig) - ctx.Given(`^\{(.*)\} http config as below`, control.AddHTTPConfig) - ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig) - ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) - - // When - ctx.When(`^generate \{(\d+)\} regex logs, with interval \{(\d+)\}ms$`, trigger.RegexSingle) - ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) - ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) - ctx.When(`^add k8s label \{(.*)\}`, control.AddLabel) - ctx.When(`^remove k8s label \{(.*)\}`, control.RemoveLabel) - ctx.When(`^start docker-compose \{(\S+)\}`, setup.StartDockerComposeEnv) - ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) - - // Then - ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount) - ctx.Then(`^there is at least \{(\d+)\} logs$`, verify.LogCountAtLeast) - ctx.Then(`^there is at least \{(\d+)\} logs with filter key \{(.*)\} value \{(.*)\}$`, verify.LogCountAtLeastWithFilter) - ctx.Then(`^the log fields match regex single`, verify.RegexSingle) - ctx.Then(`^the log fields match kv`, verify.LogFieldKV) - ctx.Then(`^the log tags match kv`, verify.TagKV) - ctx.Then(`^the context of log is valid$`, verify.LogContext) - ctx.Then(`^the log fields match`, verify.LogField) - ctx.Then(`^the log labels match`, verify.LogLabel) - ctx.Then(`^the logtail log contains \{(\d+)\} times of \{(.*)\}$`, verify.LogtailPluginLog) - ctx.Then(`wait \{(\d+)\} seconds`, func(ctx context.Context, t int) context.Context { - time.Sleep(time.Duration(t) * time.Second) - return ctx - }) - - // Cleanup - ctx.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { - cleanup.HandleSignal() - return ctx, nil - }) - ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { - cleanup.All() - return ctx, nil - }) -} diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 951b42cbe8..c5252b4ac8 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -5,7 +5,7 @@ Feature: performance file to blackhole filebeat @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeFilebeat Given {docker-compose} environment - Given docker-compose type {benchmark} + Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 712f28aa09..3bb242492e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -5,7 +5,7 @@ Feature: performance file to blackhole fluentbit @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeFluentbit Given {docker-compose} environment - Given docker-compose type {benchmark} + Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 8c416bd168..c6d2100b27 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -5,7 +5,7 @@ Feature: performance file to blackhole iLogtail @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeiLogtail Given {docker-compose} environment - Given docker-compose type {e2e} + Given docker-compose boot type {e2e} Given {performance-file-to-blackhole-ilogtail-case} local config as below """ enable: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index e76cf6bc23..95db46cd40 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -5,7 +5,7 @@ Feature: performance file to blackhole vector @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeVector Given {docker-compose} environment - Given docker-compose type {benchmark} + Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template diff --git a/test/engine/cleanup/monitor.go b/test/engine/cleanup/monitor.go deleted file mode 100644 index b541b12850..0000000000 --- a/test/engine/cleanup/monitor.go +++ /dev/null @@ -1,11 +0,0 @@ -package cleanup - -import ( - "context" - - "github.com/alibaba/ilogtail/test/engine/setup/monitor" -) - -func StopMonitor(ctx context.Context) (context.Context, error) { - return monitor.StopMonitor(ctx) -} diff --git a/test/engine/setup/controller/docker_compose_boot.go b/test/engine/setup/controller/docker_compose_boot.go index 394affe08d..ee1aef82ec 100644 --- a/test/engine/setup/controller/docker_compose_boot.go +++ b/test/engine/setup/controller/docker_compose_boot.go @@ -27,9 +27,9 @@ import ( type BootController struct { } -func (c *BootController) Init(initType string) error { +func (c *BootController) Init(bootType dockercompose.DockerComposeBootType) error { logger.Info(context.Background(), "boot controller is initializing....") - return dockercompose.Load(initType) + return dockercompose.Load(bootType) } func (c *BootController) Start(ctx context.Context) error { diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index b3a4c7cf52..289ba41e32 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -12,18 +12,25 @@ import ( "github.com/alibaba/ilogtail/pkg/protocol" "github.com/alibaba/ilogtail/test/config" "github.com/alibaba/ilogtail/test/engine/setup/controller" + "github.com/alibaba/ilogtail/test/engine/setup/dockercompose" ) const dependencyHome = "test_cases" -var dockerComposeType = "e2e" - type DockerComposeEnv struct { BootController *controller.BootController + BootType dockercompose.DockerComposeBootType } -func SetDockerComposeType(t string) { - dockerComposeType = t +func SetDockerComposeBootType(t dockercompose.DockerComposeBootType) error { + if dockerComposeEnv, ok := Env.(*DockerComposeEnv); ok { + if t != dockercompose.DockerComposeBootTypeE2E && t != dockercompose.DockerComposeBootTypeBenchmark { + return fmt.Errorf("invalid docker compose boot type, not e2e or benchmark") + } + dockerComposeEnv.BootType = t + return nil + } + return fmt.Errorf("env is not docker-compose") } func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context.Context, error) { @@ -34,7 +41,7 @@ func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context. return ctx, err } dockerComposeEnv.BootController = new(controller.BootController) - if err = dockerComposeEnv.BootController.Init(dockerComposeType); err != nil { + if err = dockerComposeEnv.BootController.Init(dockerComposeEnv.BootType); err != nil { return ctx, err } diff --git a/test/engine/setup/dockercompose/common.go b/test/engine/setup/dockercompose/common.go index e7d64122e7..edba031731 100644 --- a/test/engine/setup/dockercompose/common.go +++ b/test/engine/setup/dockercompose/common.go @@ -20,9 +20,11 @@ import ( "sync" ) +type DockerComposeBootType = string + const ( - dockerComposeTypeE2E = "e2e" - dockerComposeTypeBenchmark = "benchmark" + DockerComposeBootTypeE2E DockerComposeBootType = "e2e" + DockerComposeBootTypeBenchmark DockerComposeBootType = "benchmark" ) var networkMapping = make(map[string]string) @@ -38,13 +40,13 @@ type Booter interface { } // Load configuration to the Booter. -func Load(loadType string) error { +func Load(bootType DockerComposeBootType) error { mu.Lock() defer mu.Unlock() - switch loadType { - case dockerComposeTypeE2E: + switch bootType { + case DockerComposeBootTypeE2E: instance = NewComposeBooter() - case dockerComposeTypeBenchmark: + case DockerComposeBootTypeBenchmark: instance = NewComposeBenchmarkBooter() default: return errors.New("invalid load type") diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index f82607db4c..266847aacd 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,6 +27,7 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -103,6 +104,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { + monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index c8898d95ca..66f07cfb0c 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -49,11 +49,11 @@ func StartMonitor(ctx context.Context, containerName string) (context.Context, e return ctx, err } -func StopMonitor(ctx context.Context) (context.Context, error) { +func StopMonitor() error { if isMonitoring.Load() { stopCh <- true } - return ctx, nil + return nil } func monitoring(client *client.Client, containerName string) { diff --git a/test/engine/steps.go b/test/engine/steps.go index 8e5e0d59ee..d2f6f6f08c 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -10,6 +10,7 @@ import ( "github.com/alibaba/ilogtail/test/engine/cleanup" "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" "github.com/alibaba/ilogtail/test/engine/trigger" "github.com/alibaba/ilogtail/test/engine/verify" @@ -26,11 +27,13 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig) ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) ctx.Given(`^mkdir \{(.*)\}`, setup.Mkdir) + ctx.Given(`^docker-compose boot type \{(\S+)\}$`, setup.SetDockerComposeBootType) // When ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingle) ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingleGBK) ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) + ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) ctx.When(`^add k8s label \{(.*)\}`, control.AddLabel) ctx.When(`^remove k8s label \{(.*)\}`, control.RemoveLabel) ctx.When(`^start docker-compose \{(\S+)\}`, setup.StartDockerComposeEnv) @@ -40,6 +43,7 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^query through \{(.*)\}`, control.SetQuery) ctx.When(`^begin trigger`, trigger.BeginTrigger) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) + ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) // Then ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount) From c8932d3affdec70dd6544e8a9c66b89a7410ed69 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 05:13:41 -0700 Subject: [PATCH 26/88] polish --- test/engine/cleanup/helper.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index 503da0de9e..0a9408e749 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -47,7 +47,6 @@ func All() { _, _ = control.RemoveAllLocalConfig(ctx) _, _ = AllGeneratedLog(ctx) _, _ = GoTestCache(ctx) - _, _ = StopMonitor(ctx) _, _ = DeleteContainers(ctx) // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { From 0ced77f2e5d28f14f653097d12abd553ba2f906c Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 05:27:28 -0700 Subject: [PATCH 27/88] polish --- .github/workflows/benchmark.yaml | 45 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index f940790479..3b26751ca1 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,11 +26,8 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - types: - - closed - push: - branches: - - benchmark + # types: + # - closed jobs: CI: @@ -85,11 +82,11 @@ jobs: WITHOUTGDB: ON run: make benchmark - # - name: Download previous benchmark data - # uses: actions/cache@v4 - # with: - # path: ./cache - # key: ${{ runner.os }}-benchmark + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 @@ -97,26 +94,26 @@ jobs: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" - # external-data-json-path: ./cache/ilogtail_statistic.json + external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false summary-always: true - - name: Store benchmark result other agents - run: | - git checkout gh-pages - cp test/benchmark/report/records_all.json dev/bench/records_all.js - sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js - git add dev/bench/records_all.js - last_commit_message=$(git log -1 --pretty=%B) - git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" + # - name: Store benchmark result other agents + # run: | + # git checkout gh-pages + # cp test/benchmark/report/records_all.json dev/bench/records_all.js + # sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js + # git add dev/bench/records_all.js + # last_commit_message=$(git log -1 --pretty=%B) + # git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # Push gh-pages branch by yourself - - name: Push benchmark result - # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages - run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages + # # Push gh-pages branch by yourself + # - name: Push benchmark result + # # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + # run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages result: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] From 6287cdb55d15c37e37ea6439b8e4e0fe16ec1a63 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 05:42:33 -0700 Subject: [PATCH 28/88] resolve lint --- test/engine/setup/controller/docker_compose_boot.go | 2 +- test/engine/setup/docker_compose.go | 4 ++-- test/engine/setup/dockercompose/common.go | 8 ++++---- test/engine/setup/dockercompose/compose_benchmark.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/engine/setup/controller/docker_compose_boot.go b/test/engine/setup/controller/docker_compose_boot.go index ee1aef82ec..dff979a41f 100644 --- a/test/engine/setup/controller/docker_compose_boot.go +++ b/test/engine/setup/controller/docker_compose_boot.go @@ -27,7 +27,7 @@ import ( type BootController struct { } -func (c *BootController) Init(bootType dockercompose.DockerComposeBootType) error { +func (c *BootController) Init(bootType dockercompose.BootType) error { logger.Info(context.Background(), "boot controller is initializing....") return dockercompose.Load(bootType) } diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 289ba41e32..7aec0d57a0 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -19,10 +19,10 @@ const dependencyHome = "test_cases" type DockerComposeEnv struct { BootController *controller.BootController - BootType dockercompose.DockerComposeBootType + BootType dockercompose.BootType } -func SetDockerComposeBootType(t dockercompose.DockerComposeBootType) error { +func SetDockerComposeBootType(t dockercompose.BootType) error { if dockerComposeEnv, ok := Env.(*DockerComposeEnv); ok { if t != dockercompose.DockerComposeBootTypeE2E && t != dockercompose.DockerComposeBootTypeBenchmark { return fmt.Errorf("invalid docker compose boot type, not e2e or benchmark") diff --git a/test/engine/setup/dockercompose/common.go b/test/engine/setup/dockercompose/common.go index edba031731..867ac55dd2 100644 --- a/test/engine/setup/dockercompose/common.go +++ b/test/engine/setup/dockercompose/common.go @@ -20,11 +20,11 @@ import ( "sync" ) -type DockerComposeBootType = string +type BootType = string const ( - DockerComposeBootTypeE2E DockerComposeBootType = "e2e" - DockerComposeBootTypeBenchmark DockerComposeBootType = "benchmark" + DockerComposeBootTypeE2E BootType = "e2e" + DockerComposeBootTypeBenchmark BootType = "benchmark" ) var networkMapping = make(map[string]string) @@ -40,7 +40,7 @@ type Booter interface { } // Load configuration to the Booter. -func Load(bootType DockerComposeBootType) error { +func Load(bootType BootType) error { mu.Lock() defer mu.Unlock() switch bootType { diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index 266847aacd..5d89cbc1ea 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -104,7 +104,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - monitor.StopMonitor() + _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", From 6e61c0ffcd4383a1585d20ba7b8584fba2893c55 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 05:55:50 -0700 Subject: [PATCH 29/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 40 +++++++++++++------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 3b26751ca1..53ed119871 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,12 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - # types: - # - closed + types: + - closed jobs: CI: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -82,41 +82,33 @@ jobs: WITHOUTGDB: ON run: make benchmark - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" - external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false summary-always: true - # - name: Store benchmark result other agents - # run: | - # git checkout gh-pages - # cp test/benchmark/report/records_all.json dev/bench/records_all.js - # sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js - # git add dev/bench/records_all.js - # last_commit_message=$(git log -1 --pretty=%B) - # git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" + - name: Store benchmark result other agents + run: | + git checkout gh-pages + cp test/benchmark/report/records_all.json dev/bench/records_all.js + sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js + git add dev/bench/records_all.js + last_commit_message=$(git log -1 --pretty=%B) + git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # # Push gh-pages branch by yourself - # - name: Push benchmark result - # # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages - # run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages + # Push result to branch gh-pages + - name: Push benchmark result + run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages result: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] steps: - name: Build Result - run: echo "Just to make the GitHub merge button green" + run: echo "Run benchmark succeeded" From 087e49774da7786144deef04499b1c62cfc708f2 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 07:08:20 -0700 Subject: [PATCH 30/88] fix bug --- test/engine/setup/docker_compose.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 7aec0d57a0..2c85bbbec9 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -106,6 +106,7 @@ func NewDockerComposeEnv() *DockerComposeEnv { reportDir := root + "/report/" _ = os.Mkdir(reportDir, 0750) config.ConfigDir = reportDir + "config" + env.BootType = dockercompose.DockerComposeBootTypeE2E return env } From e7d34617790a83d840bdd68a7674af1826e52047 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 18:42:04 -0700 Subject: [PATCH 31/88] update benchmark ilogtail --- .../case.feature | 27 ++----------------- .../docker-compose.yaml | 21 ++++++--------- .../ilogtail.yaml | 19 +++++++++++++ 3 files changed, 29 insertions(+), 38 deletions(-) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index c6d2100b27..684b6127be 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -5,33 +5,10 @@ Feature: performance file to blackhole iLogtail @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeiLogtail Given {docker-compose} environment - Given docker-compose boot type {e2e} - Given {performance-file-to-blackhole-ilogtail-case} local config as below - """ - enable: true - inputs: - - Type: input_file - FilePaths: - - /home/test-log/*.log - processors: - - Type: processor_parse_json_native - SourceKey: content - - Type: processor_filter_regex_native - FilterKey: - - user-agent - FilterRegex: - - ^no-agent$ - flushers: - - Type: flusher_sls - Region: cn-hangzhou - Endpoint: cn-hangzhou.log.aliyuncs.com - Project: test_project - Logstore: test_logstore - """ - Given iLogtail container mount {.} to {/home/test-log} + Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index df9ceffb57..010bef3b17 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -15,17 +15,12 @@ version: '3.8' services: - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.49.1 + ilogtailC: + image: aliyun/loongcollector:0.0.1 volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro - ports: - - "8080:8080" - privileged: true - devices: - - /dev/kmsg - restart: always + - .:/ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - .:/home/ilogtail + healthcheck: + test: "cat /loongcollector/log/loongcollector.LOG" + interval: 15s + timeout: 5s diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml new file mode 100644 index 0000000000..3bde206ddf --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml @@ -0,0 +1,19 @@ +enable: true +inputs: + - Type: input_file + FilePaths: + - /home/ilogtail/*.log +processors: + - Type: processor_parse_json_native + SourceKey: content + - Type: processor_filter_regex_native + FilterKey: + - user-agent + FilterRegex: + - ^no-agent$ +flushers: + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore \ No newline at end of file From 9bcc8ad7a024ac48075d92e38fdefbf3fed4c4e2 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 18:57:33 -0700 Subject: [PATCH 32/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 53ed119871..27cde8007f 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,12 +26,15 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - types: - - closed + # types: + # - closed + push: + branches: + - benchmark jobs: CI: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -82,12 +85,19 @@ jobs: WITHOUTGDB: ON run: make benchmark + # - name: Download previous benchmark data + # uses: actions/cache@v4 + # with: + # path: ./cache + # key: ${{ runner.os }}-benchmark + - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" + # external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false summary-always: true @@ -100,9 +110,10 @@ jobs: last_commit_message=$(git log -1 --pretty=%B) git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # Push result to branch gh-pages + # Push gh-pages branch by yourself - name: Push benchmark result - run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages result: if: github.event.pull_request.merged == true @@ -111,4 +122,4 @@ jobs: needs: [ CI ] steps: - name: Build Result - run: echo "Run benchmark succeeded" + run: echo "Just to make the GitHub merge button green" From d586689d620a964b3e3133817d3aff9a8b8470a5 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 19:00:53 -0700 Subject: [PATCH 33/88] update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 48d8f8f436..c98f429dfc 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ e2e-core: clean gocdocker e2edocker ./scripts/e2e.sh e2e core .PHONY: e2e-performance -e2e-performance: clean docker gocdocker +e2e-performance: clean e2edocker ./scripts/e2e.sh e2e performance .PHONY: unittest_e2e_engine From 115580d864c7a16c2ffbdae10f8b19d29a79bf69 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 19:08:31 -0700 Subject: [PATCH 34/88] polish --- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 010bef3b17..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - .:/ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" From 0a7a5ef5910839ab65307d00bdba8e3110be239d Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 19:16:55 -0700 Subject: [PATCH 35/88] fix bug --- test/engine/cleanup/helper.go | 3 +++ test/engine/setup/dockercompose/compose_benchmark.go | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index 0a9408e749..df3513c298 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -22,6 +22,7 @@ import ( "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" ) @@ -48,6 +49,8 @@ func All() { _, _ = AllGeneratedLog(ctx) _, _ = GoTestCache(ctx) _, _ = DeleteContainers(ctx) + // Stop Monitor + _ = monitor.StopMonitor() // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index 5d89cbc1ea..f82607db4c 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,7 +27,6 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -104,7 +103,6 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", From 6402c1d5fbe1656d71699e121f24231b1fbc2afb Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 15 Oct 2024 19:22:19 -0700 Subject: [PATCH 36/88] polish --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c98f429dfc..2d3c129d12 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ e2e-core: clean gocdocker e2edocker ./scripts/e2e.sh e2e core .PHONY: e2e-performance -e2e-performance: clean e2edocker +e2e-performance: clean docker ./scripts/e2e.sh e2e performance .PHONY: unittest_e2e_engine @@ -212,7 +212,7 @@ unittest_pluginmanager: clean import_plugins # benchmark .PHONY: benchmark -benchmark: clean gocdocker e2edocker +benchmark: clean e2edocker ./scripts/e2e.sh benchmark performance ./scripts/benchmark_collect_result.sh From bf80fd6a79807f65c59f8443aedc0e3a4653b415 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 02:03:51 -0700 Subject: [PATCH 37/88] polish --- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index f5f31279dc..97bf5deb2e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" From 4f44830c03a61919d1bd4da21efce963824883a9 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 04:15:01 -0700 Subject: [PATCH 38/88] increase log generate speed --- .../performance_file_to_blackhole_filebeat/case.feature | 2 +- .../performance_file_to_blackhole_fluentbit/case.feature | 2 +- .../performance_file_to_blackhole_ilogtail/case.feature | 2 +- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 1 + .../performance_file_to_blackhole_vector/case.feature | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index c5252b4ac8..2222a791ac 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 3bb242492e..4bd33b8c29 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 684b6127be..1b1f7d03c6 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 97bf5deb2e..c208bd3083 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -19,6 +19,7 @@ services: image: aliyun/loongcollector:0.0.1 volumes: - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 95db46cd40..e531c58c72 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ From 4fe624996c3854b72a13fb200e47b30b7d517744 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 05:14:22 -0700 Subject: [PATCH 39/88] test --- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index c208bd3083..a5d012aacf 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + # - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: From 311837d63cfbb58345924fe393c2997953df4c09 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 06:04:10 -0700 Subject: [PATCH 40/88] update ilogtail docker-compose.yaml --- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index a5d012aacf..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,6 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - # - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: From 0d9dfe240656bd066f092af8006a33a96f2769b8 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 06:07:30 -0700 Subject: [PATCH 41/88] polish --- .github/workflows/benchmark.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 27cde8007f..bfcd419eb1 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,15 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - # types: - # - closed - push: - branches: - - benchmark + types: + - closed jobs: CI: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -85,19 +82,12 @@ jobs: WITHOUTGDB: ON run: make benchmark - # - name: Download previous benchmark data - # uses: actions/cache@v4 - # with: - # path: ./cache - # key: ${{ runner.os }}-benchmark - - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 with: name: benchmark tool: "customSmallerIsBetter" output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" - # external-data-json-path: ./cache/ilogtail_statistic.json auto-push: false summary-always: true @@ -112,8 +102,7 @@ jobs: # Push gh-pages branch by yourself - name: Push benchmark result - # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages - run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages + run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages result: if: github.event.pull_request.merged == true From 34d3f3f004be41fce805f303f6719ebe911513f1 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 16 Oct 2024 19:53:21 -0700 Subject: [PATCH 42/88] update doc --- docs/cn/developer-guide/test/benchmark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index 9030f51b01..d73ff101ac 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -38,7 +38,7 @@ Feature: performance file to blackhole vector - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 - `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 -- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 +- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 - `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) From 506f811e985f06be4a3221aaf65df1ea56a8b351 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 17 Oct 2024 06:15:30 -0700 Subject: [PATCH 43/88] merge to e2e-performance-test --- docs/cn/developer-guide/test/benchmark.md | 2 +- .../performance_file_to_blackhole_filebeat/case.feature | 2 +- .../performance_file_to_blackhole_fluentbit/case.feature | 2 +- .../performance_file_to_blackhole_ilogtail/case.feature | 2 +- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 2 +- .../performance_file_to_blackhole_vector/case.feature | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index 9030f51b01..d73ff101ac 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -38,7 +38,7 @@ Feature: performance file to blackhole vector - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 - `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 -- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`director`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 +- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 - `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index c5252b4ac8..2222a791ac 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 3bb242492e..4bd33b8c29 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 684b6127be..1b1f7d03c6 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 97bf5deb2e..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 95db46cd40..e531c58c72 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,7 +8,7 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template """ {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} """ From 70a422db55ad8baf9ec4d2730ea1082290176367 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 17 Oct 2024 19:22:53 -0700 Subject: [PATCH 44/88] test benchmark action with write permission --- .github/workflows/benchmark.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index bfcd419eb1..b1e526b3ee 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,12 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - types: - - closed + # types: + # - closed jobs: CI: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -39,6 +39,8 @@ jobs: go-version: [ 1.19.10 ] runner: [ ubuntu-latest ] fail-fast: true + permissions: + contents: write steps: # Clean up space to prevent action from running out of disk space. - name: clean @@ -105,7 +107,7 @@ jobs: run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages result: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] From 9fdacf56bba980c68c755d84243647444dddfa7d Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 00:42:47 -0700 Subject: [PATCH 45/88] fix bug --- test/engine/cleanup/helper.go | 3 --- test/engine/setup/dockercompose/compose_benchmark.go | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index df3513c298..0a9408e749 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -22,7 +22,6 @@ import ( "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" ) @@ -49,8 +48,6 @@ func All() { _, _ = AllGeneratedLog(ctx) _, _ = GoTestCache(ctx) _, _ = DeleteContainers(ctx) - // Stop Monitor - _ = monitor.StopMonitor() // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index f82607db4c..5d89cbc1ea 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,6 +27,7 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -103,6 +104,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { + _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", From b3c10cd3531d35b015c85cffb5dccda8a5dd595d Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 00:44:47 -0700 Subject: [PATCH 46/88] update log generator and monitor --- .../case.feature | 6 +- .../case.feature | 6 +- .../case.feature | 6 +- .../case.feature | 6 +- test/engine/cleanup/helper.go | 3 - .../setup/dockercompose/compose_benchmark.go | 3 + test/engine/setup/monitor/monitor.go | 80 +++++++++++++++---- test/engine/setup/monitor/statistic.go | 80 +++++++++++++++---- test/engine/steps.go | 4 +- test/engine/trigger/file.go | 50 ++++++++++++ 10 files changed, 193 insertions(+), 51 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 2222a791ac..8fe7361afe 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,7 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 4bd33b8c29..18e00cc4b5 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,7 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 1b1f7d03c6..c3e8a6f7cf 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,7 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ \ No newline at end of file + When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When stop monitor in {30} seconds and verify if log processing is finished \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index e531c58c72..0b63228ee2 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,7 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index df3513c298..0a9408e749 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -22,7 +22,6 @@ import ( "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" ) @@ -49,8 +48,6 @@ func All() { _, _ = AllGeneratedLog(ctx) _, _ = GoTestCache(ctx) _, _ = DeleteContainers(ctx) - // Stop Monitor - _ = monitor.StopMonitor() // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index f82607db4c..4eb2ad3f39 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,6 +27,7 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -103,6 +104,8 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { + // Stop Monitor + _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 66f07cfb0c..8fcfeccd07 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -5,7 +5,9 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" + "sync" "sync/atomic" "time" @@ -20,10 +22,50 @@ const ( interval = 3 ) -var stopCh chan bool -var isMonitoring atomic.Bool +var monitor Monitor + +type Monitor struct { + ctx context.Context + stopCh chan bool + isMonitoring atomic.Bool + statistic *Statistic + mu sync.Mutex +} func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { + return monitor.Start(ctx, containerName) +} + +func StopMonitor() error { + return monitor.Stop() +} + +func StopMonitorAndVerifyFinished(ctx context.Context, timeout int) (context.Context, error) { + time.Sleep(time.Duration(timeout) * time.Second) + cpuRawData := monitor.getCpuRawData() + lastCpuRawData := cpuRawData[len(cpuRawData)-1] + + // Step 1: Sort the data + sort.Float64s(cpuRawData) + + // Step 2: Calculate Q1 and Q3 + Q1 := cpuRawData[len(cpuRawData)/4] + Q3 := cpuRawData[3*len(cpuRawData)/4] + + // Step 3: Calculate IQR + IQR := Q3 - Q1 + + // Step 4: Determine the lower bounds for outliers + lowerBound := Q1 - 1.5*IQR + + // Step 5: Find out if the outliers exist at the tail + if lastCpuRawData > lowerBound { + return ctx, fmt.Errorf("Benchmark not finished, CPU usage is still high") + } + return ctx, nil +} + +func (m *Monitor) Start(ctx context.Context, containerName string) (context.Context, error) { // connect to cadvisor client, err := client.NewClient("http://localhost:8080/") if err != nil { @@ -38,10 +80,10 @@ func StartMonitor(ctx context.Context, containerName string) (context.Context, e for _, container := range allContainers { containerFullName := container.Aliases[0] if strings.Contains(containerFullName, containerName) { - stopCh = make(chan bool) - isMonitoring.Store(true) + m.stopCh = make(chan bool) + m.isMonitoring.Store(true) fmt.Println("Start monitoring container:", containerFullName) - go monitoring(client, containerFullName) + go m.monitoring(client, containerFullName) return ctx, nil } } @@ -49,14 +91,14 @@ func StartMonitor(ctx context.Context, containerName string) (context.Context, e return ctx, err } -func StopMonitor() error { - if isMonitoring.Load() { - stopCh <- true +func (m *Monitor) Stop() error { + if m.isMonitoring.Load() { + m.stopCh <- true } return nil } -func monitoring(client *client.Client, containerName string) { +func (m *Monitor) monitoring(client *client.Client, containerName string) { // create csv file root, _ := filepath.Abs(".") reportDir := root + "/report/" @@ -67,14 +109,14 @@ func monitoring(client *client.Client, containerName string) { defer ticker.Stop() // read from cadvisor per interval seconds request := &v1.ContainerInfoRequest{NumStats: 10} - monitorStatistic := NewMonitorStatistic(config.CaseName) + m.statistic = NewMonitorStatistic(config.CaseName) for { select { - case <-stopCh: - isMonitoring.Store(false) - bytes, _ := monitorStatistic.MarshalStatisticJSON() + case <-m.stopCh: + m.isMonitoring.Store(false) + bytes, _ := m.statistic.MarshalStatisticJSON() _ = os.WriteFile(statisticFile, bytes, 0600) - bytes, _ = monitorStatistic.MarshalRecordsJSON() + bytes, _ = m.statistic.MarshalRecordsJSON() _ = os.WriteFile(recordsFile, bytes, 0600) return case <-ticker.C: @@ -84,9 +126,17 @@ func monitoring(client *client.Client, containerName string) { fmt.Println("Error getting container info:", err) return } + m.mu.Lock() for _, stat := range containerInfo.Stats { - monitorStatistic.UpdateStatistic(stat) + m.statistic.UpdateStatistic(stat) } + m.mu.Unlock() } } } + +func (m *Monitor) getCpuRawData() []float64 { + m.mu.Lock() + defer m.mu.Unlock() + return m.statistic.GetCpuRawData() +} diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index 83ea2934b7..bd98357dfe 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -3,30 +3,64 @@ package monitor import ( "encoding/json" "fmt" + "sort" v1 "github.com/google/cadvisor/info/v1" ) type Info struct { - maxVal float64 - avgVal float64 - cnt int64 values []float64 } func (s *Info) Add(val float64) { s.values = append(s.values, val) - if s.cnt < 1 { - s.maxVal = val - s.avgVal = val - s.cnt++ - return +} + +func (s *Info) CalculateMax() float64 { + maxVal := 0.0 + for _, val := range s.values { + if val > maxVal { + maxVal = val + } } - if s.maxVal < val { - s.maxVal = val + return maxVal +} + +func (s *Info) CalculateAvg() float64 { + values_copy := make([]float64, len(s.values)) + for i, val := range s.values { + values_copy[i] = val + } + + // Step 1: Sort the values_copy + sort.Float64s(values_copy) + + // Step 2: Calculate Q1 and Q3 + Q1 := values_copy[len(values_copy)/4] + Q3 := values_copy[3*len(values_copy)/4] + + // Step 3: Calculate IQR + IQR := Q3 - Q1 + + // Step 4: Determine the lower and upper bounds for outliers + lowerBound := Q1 - 1.5*IQR + upperBound := Q3 + 1.5*IQR + + // Step 5: Filter out the outliers + cnt := 0 + avg := 0.0 + for _, value := range values_copy { + if value >= lowerBound && value <= upperBound { + if cnt == 0 { + avg = value + } else { + avg = float64(cnt)/float64(cnt+1)*avg + float64(value)/float64(cnt+1) + } + cnt++ + } } - s.avgVal = float64(s.cnt)/float64(s.cnt+1)*s.avgVal + val/float64(s.cnt+1) - s.cnt++ + + return avg } type Statistic struct { @@ -56,6 +90,20 @@ func (m *Statistic) UpdateStatistic(stat *v1.ContainerStats) { // fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) } +func (m *Statistic) ClearStatistic() { + m.cpu.values = m.cpu.values[:0] + m.mem.values = m.mem.values[:0] + m.lastStat = nil +} + +func (m *Statistic) GetCpuRawData() []float64 { + cpuRawData := make([]float64, len(m.cpu.values)) + for i, val := range m.cpu.values { + cpuRawData[i] = val + } + return cpuRawData +} + func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { if lastStat == nil { return 0 @@ -73,10 +121,10 @@ type StatisticItem struct { func (m *Statistic) MarshalStatisticJSON() ([]byte, error) { items := []StatisticItem{ - {"CPU_Usage_Max - " + m.name, m.cpu.maxVal, "%"}, - {"CPU_Usage_Avg - " + m.name, m.cpu.avgVal, "%"}, - {"Memory_Usage_Max - " + m.name, m.mem.maxVal, "MB"}, - {"Memory_Usage_Avg - " + m.name, m.mem.avgVal, "MB"}, + {"CPU_Usage_Max - " + m.name, m.cpu.CalculateMax(), "%"}, + {"CPU_Usage_Avg - " + m.name, m.cpu.CalculateAvg(), "%"}, + {"Memory_Usage_Max - " + m.name, m.mem.CalculateMax(), "MB"}, + {"Memory_Usage_Avg - " + m.name, m.mem.CalculateAvg(), "MB"}, } // Serialize the slice to JSON diff --git a/test/engine/steps.go b/test/engine/steps.go index d2f6f6f08c..75bdbab491 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -33,7 +33,6 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingle) ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingleGBK) ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) - ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) ctx.When(`^add k8s label \{(.*)\}`, control.AddLabel) ctx.When(`^remove k8s label \{(.*)\}`, control.RemoveLabel) ctx.When(`^start docker-compose \{(\S+)\}`, setup.StartDockerComposeEnv) @@ -43,7 +42,10 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^query through \{(.*)\}`, control.SetQuery) ctx.When(`^begin trigger`, trigger.BeginTrigger) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) + ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) + ctx.When(`^generate random json logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomJsonLogToFile) ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) + ctx.When(`^stop monitor in \{(\d+)\} seconds and verify if log processing is finished$`, monitor.StopMonitorAndVerifyFinished) // Then ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index aaa47deb3f..57b697f0b4 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -2,6 +2,7 @@ package trigger import ( "context" + "math/rand" "os" "path/filepath" "time" @@ -41,3 +42,52 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t } } } + +func GenerateRandomJsonLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { + // json template + jsonTemplates := []string{ + `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}`, + `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}`, + `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}`, + `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}`, + `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}`, + } + maxLen := 0 + for i := 0; i < len(jsonTemplates); i++ { + if len(jsonTemplates[i]) > maxLen { + maxLen = len(jsonTemplates[i]) + } + } + + // clear file + path = filepath.Join(config.CaseHome, path) + path = filepath.Clean(path) + _ = os.WriteFile(path, []byte{}, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 + + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) + + timeout := time.After(time.Minute * time.Duration(totalTime)) + + rand.Seed(time.Now().UnixNano()) + + for { + select { + // context is done + case <-ctx.Done(): + // clear file + _ = file.Close() + return ctx, ctx.Err() + // all time is done + case <-timeout: + // clear file + _ = file.Close() + return ctx, nil + default: + if limiter.AllowN(time.Now(), maxLen) { + randomIndex := rand.Intn(len(jsonTemplates)) + _, _ = file.WriteString(jsonTemplates[randomIndex] + "\n") + } + } + } +} From b07d0fa99c12746ce2ff6a198a480efd4779eefe Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 01:16:44 -0700 Subject: [PATCH 47/88] resolve lint --- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- test/engine/setup/monitor/monitor.go | 11 +++++------ test/engine/setup/monitor/statistic.go | 10 +++------- test/engine/trigger/file.go | 16 ++++++++-------- 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 8fe7361afe..06dd214b2e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 18e00cc4b5..5ba3327095 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index c3e8a6f7cf..eb48fa48b1 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When stop monitor in {30} seconds and verify if log processing is finished \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 0b63228ee2..c477b0c25f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate random json logs to file, speed {100}MB/s, total {1}min, to file {./a.log} + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When stop monitor in {30} seconds and verify if log processing is finished diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 8fcfeccd07..70896f40bd 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -25,7 +25,6 @@ const ( var monitor Monitor type Monitor struct { - ctx context.Context stopCh chan bool isMonitoring atomic.Bool statistic *Statistic @@ -42,8 +41,8 @@ func StopMonitor() error { func StopMonitorAndVerifyFinished(ctx context.Context, timeout int) (context.Context, error) { time.Sleep(time.Duration(timeout) * time.Second) - cpuRawData := monitor.getCpuRawData() - lastCpuRawData := cpuRawData[len(cpuRawData)-1] + cpuRawData := monitor.getCPURawData() + lastCPURawData := cpuRawData[len(cpuRawData)-1] // Step 1: Sort the data sort.Float64s(cpuRawData) @@ -59,7 +58,7 @@ func StopMonitorAndVerifyFinished(ctx context.Context, timeout int) (context.Con lowerBound := Q1 - 1.5*IQR // Step 5: Find out if the outliers exist at the tail - if lastCpuRawData > lowerBound { + if lastCPURawData > lowerBound { return ctx, fmt.Errorf("Benchmark not finished, CPU usage is still high") } return ctx, nil @@ -135,8 +134,8 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { } } -func (m *Monitor) getCpuRawData() []float64 { +func (m *Monitor) getCPURawData() []float64 { m.mu.Lock() defer m.mu.Unlock() - return m.statistic.GetCpuRawData() + return m.statistic.GetCPURawData() } diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index bd98357dfe..a73a2f10cc 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -28,9 +28,7 @@ func (s *Info) CalculateMax() float64 { func (s *Info) CalculateAvg() float64 { values_copy := make([]float64, len(s.values)) - for i, val := range s.values { - values_copy[i] = val - } + copy(values_copy, s.values) // Step 1: Sort the values_copy sort.Float64s(values_copy) @@ -54,7 +52,7 @@ func (s *Info) CalculateAvg() float64 { if cnt == 0 { avg = value } else { - avg = float64(cnt)/float64(cnt+1)*avg + float64(value)/float64(cnt+1) + avg = float64(cnt)/float64(cnt+1)*avg + value/float64(cnt+1) } cnt++ } @@ -98,9 +96,7 @@ func (m *Statistic) ClearStatistic() { func (m *Statistic) GetCpuRawData() []float64 { cpuRawData := make([]float64, len(m.cpu.values)) - for i, val := range m.cpu.values { - cpuRawData[i] = val - } + copy(cpuRawData, m.cpu.values) return cpuRawData } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 57b697f0b4..67e0e7f172 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -43,9 +43,9 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t } } -func GenerateRandomJsonLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - // json template - jsonTemplates := []string{ +func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { + // JSON template + JSONTemplates := []string{ `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}`, `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}`, `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}`, @@ -53,9 +53,9 @@ func GenerateRandomJsonLogToFile(ctx context.Context, speed, totalTime int, path `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}`, } maxLen := 0 - for i := 0; i < len(jsonTemplates); i++ { - if len(jsonTemplates[i]) > maxLen { - maxLen = len(jsonTemplates[i]) + for i := 0; i < len(JSONTemplates); i++ { + if len(JSONTemplates[i]) > maxLen { + maxLen = len(JSONTemplates[i]) } } @@ -85,8 +85,8 @@ func GenerateRandomJsonLogToFile(ctx context.Context, speed, totalTime int, path return ctx, nil default: if limiter.AllowN(time.Now(), maxLen) { - randomIndex := rand.Intn(len(jsonTemplates)) - _, _ = file.WriteString(jsonTemplates[randomIndex] + "\n") + randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 + _, _ = file.WriteString(JSONTemplates[randomIndex] + "\n") } } } From 546be74addcb5741607763d702410083e0c86411 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 01:36:56 -0700 Subject: [PATCH 48/88] empty commit From 267e012180c5c834bc93fb7385a3d75367b1a4fd Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 02:24:32 -0700 Subject: [PATCH 49/88] update monitor --- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../setup/dockercompose/compose_benchmark.go | 3 - test/engine/setup/monitor/monitor.go | 84 +++++++------------ test/engine/setup/monitor/statistic.go | 6 +- test/engine/steps.go | 4 +- 8 files changed, 39 insertions(+), 66 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 06dd214b2e..8d5e5fc99c 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -9,4 +9,4 @@ Feature: performance file to blackhole filebeat When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} - When stop monitor in {30} seconds and verify if log processing is finished + When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 5ba3327095..0d6c72174f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -9,4 +9,4 @@ Feature: performance file to blackhole fluentbit When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} - When stop monitor in {30} seconds and verify if log processing is finished + When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index eb48fa48b1..11d7fec4e5 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -9,4 +9,4 @@ Feature: performance file to blackhole iLogtail When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} - When stop monitor in {30} seconds and verify if log processing is finished \ No newline at end of file + When wait monitor until log processing finished \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index c477b0c25f..9ccf02bb4e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -9,4 +9,4 @@ Feature: performance file to blackhole vector When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} - When stop monitor in {30} seconds and verify if log processing is finished + When wait monitor until log processing finished diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index 4eb2ad3f39..f82607db4c 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,7 +27,6 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -104,8 +103,6 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - // Stop Monitor - _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 70896f40bd..1df921777d 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -7,7 +7,6 @@ import ( "path/filepath" "sort" "strings" - "sync" "sync/atomic" "time" @@ -25,42 +24,17 @@ const ( var monitor Monitor type Monitor struct { - stopCh chan bool isMonitoring atomic.Bool statistic *Statistic - mu sync.Mutex + stopCh chan struct{} } func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { return monitor.Start(ctx, containerName) } -func StopMonitor() error { - return monitor.Stop() -} - -func StopMonitorAndVerifyFinished(ctx context.Context, timeout int) (context.Context, error) { - time.Sleep(time.Duration(timeout) * time.Second) - cpuRawData := monitor.getCPURawData() - lastCPURawData := cpuRawData[len(cpuRawData)-1] - - // Step 1: Sort the data - sort.Float64s(cpuRawData) - - // Step 2: Calculate Q1 and Q3 - Q1 := cpuRawData[len(cpuRawData)/4] - Q3 := cpuRawData[3*len(cpuRawData)/4] - - // Step 3: Calculate IQR - IQR := Q3 - Q1 - - // Step 4: Determine the lower bounds for outliers - lowerBound := Q1 - 1.5*IQR - - // Step 5: Find out if the outliers exist at the tail - if lastCPURawData > lowerBound { - return ctx, fmt.Errorf("Benchmark not finished, CPU usage is still high") - } +func WaitMonitorUntilProcessingFinished(ctx context.Context) (context.Context, error) { + <-monitor.Done() return ctx, nil } @@ -79,8 +53,9 @@ func (m *Monitor) Start(ctx context.Context, containerName string) (context.Cont for _, container := range allContainers { containerFullName := container.Aliases[0] if strings.Contains(containerFullName, containerName) { - m.stopCh = make(chan bool) m.isMonitoring.Store(true) + m.statistic = NewMonitorStatistic(config.CaseName) + m.stopCh = make(chan struct{}) fmt.Println("Start monitoring container:", containerFullName) go m.monitoring(client, containerFullName) return ctx, nil @@ -90,34 +65,31 @@ func (m *Monitor) Start(ctx context.Context, containerName string) (context.Cont return ctx, err } -func (m *Monitor) Stop() error { - if m.isMonitoring.Load() { - m.stopCh <- true - } - return nil -} - func (m *Monitor) monitoring(client *client.Client, containerName string) { // create csv file root, _ := filepath.Abs(".") reportDir := root + "/report/" statisticFile := reportDir + config.CaseName + "_statistic.json" recordsFile := reportDir + config.CaseName + "_records.json" - // new ticker + // calculate low threshold after 60 seconds + timer := time.NewTimer(60 * time.Second) + lowThreshold := 0.0 + // read from cadvisor per interval seconds ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() - // read from cadvisor per interval seconds request := &v1.ContainerInfoRequest{NumStats: 10} - m.statistic = NewMonitorStatistic(config.CaseName) for { select { - case <-m.stopCh: - m.isMonitoring.Store(false) - bytes, _ := m.statistic.MarshalStatisticJSON() - _ = os.WriteFile(statisticFile, bytes, 0600) - bytes, _ = m.statistic.MarshalRecordsJSON() - _ = os.WriteFile(recordsFile, bytes, 0600) - return + case <-timer.C: + // 计算CPU使用率的下阈值 + cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) + copy(cpuRawData, m.statistic.GetCPURawData()) + sort.Float64s(cpuRawData) + Q1 := cpuRawData[len(cpuRawData)/4] + Q3 := cpuRawData[3*len(cpuRawData)/4] + IQR := Q3 - Q1 + lowThreshold = Q1 - 1.5*IQR + fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) case <-ticker.C: // 获取容器信息 containerInfo, err := client.DockerContainer(containerName, request) @@ -125,17 +97,23 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { fmt.Println("Error getting container info:", err) return } - m.mu.Lock() for _, stat := range containerInfo.Stats { m.statistic.UpdateStatistic(stat) } - m.mu.Unlock() + cpuRawData := m.statistic.GetCPURawData() + if (cpuRawData[len(cpuRawData)-1] < lowThreshold) && (lowThreshold > 0) { + m.isMonitoring.Store(false) + m.stopCh <- struct{}{} + bytes, _ := m.statistic.MarshalStatisticJSON() + _ = os.WriteFile(statisticFile, bytes, 0600) + bytes, _ = m.statistic.MarshalRecordsJSON() + _ = os.WriteFile(recordsFile, bytes, 0600) + return + } } } } -func (m *Monitor) getCPURawData() []float64 { - m.mu.Lock() - defer m.mu.Unlock() - return m.statistic.GetCPURawData() +func (m *Monitor) Done() <-chan struct{} { + return m.stopCh } diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index a73a2f10cc..55412df129 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -94,10 +94,8 @@ func (m *Statistic) ClearStatistic() { m.lastStat = nil } -func (m *Statistic) GetCpuRawData() []float64 { - cpuRawData := make([]float64, len(m.cpu.values)) - copy(cpuRawData, m.cpu.values) - return cpuRawData +func (m *Statistic) GetCPURawData() []float64 { + return m.cpu.values } func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { diff --git a/test/engine/steps.go b/test/engine/steps.go index 75bdbab491..9b3dea8bf7 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -43,9 +43,9 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^begin trigger`, trigger.BeginTrigger) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) - ctx.When(`^generate random json logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomJsonLogToFile) + ctx.When(`^generate random json logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomJSONLogToFile) ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) - ctx.When(`^stop monitor in \{(\d+)\} seconds and verify if log processing is finished$`, monitor.StopMonitorAndVerifyFinished) + ctx.When(`^wait monitor until log processing finished$`, monitor.WaitMonitorUntilProcessingFinished) // Then ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount) From 15815d0a3f0ca167da08894760cf19abfc60a9e3 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 03:19:07 -0700 Subject: [PATCH 50/88] empty commit From 7429e2b2c0142fb55252ddfb15953f70e541f1cb Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 03:22:52 -0700 Subject: [PATCH 51/88] update monitor --- test/engine/setup/monitor/monitor.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 1df921777d..5d1d745327 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -74,6 +74,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { // calculate low threshold after 60 seconds timer := time.NewTimer(60 * time.Second) lowThreshold := 0.0 + outlierCnt := 0 // read from cadvisor per interval seconds ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() @@ -90,6 +91,9 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { IQR := Q3 - Q1 lowThreshold = Q1 - 1.5*IQR fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) + if lowThreshold < 0 { + m.stopCh <- struct{}{} + } case <-ticker.C: // 获取容器信息 containerInfo, err := client.DockerContainer(containerName, request) @@ -102,6 +106,9 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { } cpuRawData := m.statistic.GetCPURawData() if (cpuRawData[len(cpuRawData)-1] < lowThreshold) && (lowThreshold > 0) { + outlierCnt++ + } + if outlierCnt > 5 { m.isMonitoring.Store(false) m.stopCh <- struct{}{} bytes, _ := m.statistic.MarshalStatisticJSON() From 93686e36073af778d436f5a76b3f69cfd4d1dc6f Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:28:49 -0700 Subject: [PATCH 52/88] update monitor quit logic and file trigger logic --- test/engine/setup/monitor/monitor.go | 12 ++-- test/engine/trigger/file.go | 93 ++++++++++++++++------------ 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 5d1d745327..740062367a 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -83,13 +83,16 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { select { case <-timer.C: // 计算CPU使用率的下阈值 - cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) - copy(cpuRawData, m.statistic.GetCPURawData()) + cpuRawData := make([]float64, len(m.statistic.GetCPURawData())-5) + copy(cpuRawData, m.statistic.GetCPURawData()[5:]) sort.Float64s(cpuRawData) Q1 := cpuRawData[len(cpuRawData)/4] Q3 := cpuRawData[3*len(cpuRawData)/4] IQR := Q3 - Q1 lowThreshold = Q1 - 1.5*IQR + fmt.Println("median of CPU usage rate(%):", cpuRawData[len(cpuRawData)/2]) + fmt.Println("1/4 of CPU usage rate(%):", cpuRawData[len(cpuRawData)/4]) + fmt.Println("3/4 of CPU usage rate(%):", cpuRawData[3*len(cpuRawData)/4]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) if lowThreshold < 0 { m.stopCh <- struct{}{} @@ -109,12 +112,13 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { outlierCnt++ } if outlierCnt > 5 { - m.isMonitoring.Store(false) - m.stopCh <- struct{}{} bytes, _ := m.statistic.MarshalStatisticJSON() _ = os.WriteFile(statisticFile, bytes, 0600) bytes, _ = m.statistic.MarshalRecordsJSON() _ = os.WriteFile(recordsFile, bytes, 0600) + m.isMonitoring.Store(false) + m.stopCh <- struct{}{} + m.statistic.ClearStatistic() return } } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 67e0e7f172..bc14adc1ce 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -2,9 +2,11 @@ package trigger import ( "context" + "fmt" "math/rand" "os" "path/filepath" + "sync" "time" "golang.org/x/time/rate" @@ -43,51 +45,66 @@ func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, t } } +// JSON template func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - // JSON template - JSONTemplates := []string{ - `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}`, - `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}`, - `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}`, - `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}`, - `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}`, - } - maxLen := 0 - for i := 0; i < len(JSONTemplates); i++ { - if len(JSONTemplates[i]) > maxLen { - maxLen = len(JSONTemplates[i]) - } + numGoRoutines := (speed-1)/5 + 1 + if numGoRoutines > 8 { + numGoRoutines = 8 } + wg := sync.WaitGroup{} + for i := 0; i < numGoRoutines; i++ { + wg.Add(1) + go func(ctx context.Context, thread_id, speed, totalTime int, path string) { + defer wg.Done() + JSONTemplates := []string{ + `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`, + `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`, + `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`, + `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`, + `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, + } + maxLen := 0 + for i := 0; i < len(JSONTemplates); i++ { + if len(JSONTemplates[i]) > maxLen { + maxLen = len(JSONTemplates[i]) + } + } - // clear file - path = filepath.Join(config.CaseHome, path) - path = filepath.Clean(path) - _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 + // clear file + path = filepath.Clean(path) + path = fmt.Sprintf("%d_%s", thread_id, path) + path = filepath.Join(config.CaseHome, path) + fmt.Println(path) + _ = os.WriteFile(path, []byte{}, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) - timeout := time.After(time.Minute * time.Duration(totalTime)) + timeout := time.After(time.Minute * time.Duration(totalTime)) - rand.Seed(time.Now().UnixNano()) + rand.Seed(time.Now().UnixNano()) - for { - select { - // context is done - case <-ctx.Done(): - // clear file - _ = file.Close() - return ctx, ctx.Err() - // all time is done - case <-timeout: - // clear file - _ = file.Close() - return ctx, nil - default: - if limiter.AllowN(time.Now(), maxLen) { - randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 - _, _ = file.WriteString(JSONTemplates[randomIndex] + "\n") + for { + select { + // context is done + case <-ctx.Done(): + // clear file + _ = file.Close() + return + // all time is done + case <-timeout: + // clear file + _ = file.Close() + return + default: + if limiter.AllowN(time.Now(), maxLen) { + randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 + _, _ = file.WriteString(JSONTemplates[randomIndex]) + } + } } - } + }(ctx, i, speed/numGoRoutines, totalTime, path) } + wg.Wait() + return ctx, nil } From c69d5384f3a2127b140b5997cb658d7828d86892 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:31:27 -0700 Subject: [PATCH 53/88] update file trigger --- test/engine/steps.go | 1 - test/engine/trigger/file.go | 31 ------------------------------- 2 files changed, 32 deletions(-) diff --git a/test/engine/steps.go b/test/engine/steps.go index 9b3dea8bf7..cc73b459ba 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -42,7 +42,6 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^query through \{(.*)\}`, control.SetQuery) ctx.When(`^begin trigger`, trigger.BeginTrigger) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) - ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile) ctx.When(`^generate random json logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomJSONLogToFile) ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) ctx.When(`^wait monitor until log processing finished$`, monitor.WaitMonitorUntilProcessingFinished) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index bc14adc1ce..83cfe2dfd3 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -14,37 +14,6 @@ import ( "github.com/alibaba/ilogtail/test/config" ) -func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, templateStr string) (context.Context, error) { - // clear file - path = filepath.Join(config.CaseHome, path) - path = filepath.Clean(path) - _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), len(templateStr)) - - timeout := time.After(time.Minute * time.Duration(totalTime)) - - for { - select { - // context is done - case <-ctx.Done(): - // clear file - _ = file.Close() - return ctx, ctx.Err() - // all time is done - case <-timeout: - // clear file - _ = file.Close() - return ctx, nil - default: - if limiter.AllowN(time.Now(), len(templateStr)) { - _, _ = file.WriteString(templateStr + "\n") - } - } - } -} - // JSON template func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { numGoRoutines := (speed-1)/5 + 1 From c47a2e1839384dbf7b0bdcdc014320044b743e18 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:50:54 -0700 Subject: [PATCH 54/88] resolve lint --- .../setup/dockercompose/compose_benchmark.go | 2 -- test/engine/setup/monitor/statistic.go | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test/engine/setup/dockercompose/compose_benchmark.go b/test/engine/setup/dockercompose/compose_benchmark.go index 5d89cbc1ea..f82607db4c 100644 --- a/test/engine/setup/dockercompose/compose_benchmark.go +++ b/test/engine/setup/dockercompose/compose_benchmark.go @@ -27,7 +27,6 @@ import ( "github.com/alibaba/ilogtail/pkg/logger" "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/setup/monitor" ) const ( @@ -104,7 +103,6 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - _ = monitor.StopMonitor() execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index 55412df129..d5e900a105 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -27,15 +27,15 @@ func (s *Info) CalculateMax() float64 { } func (s *Info) CalculateAvg() float64 { - values_copy := make([]float64, len(s.values)) - copy(values_copy, s.values) + valuesCopy := make([]float64, len(s.values)) + copy(valuesCopy, s.values) - // Step 1: Sort the values_copy - sort.Float64s(values_copy) + // Step 1: Sort the valuesCopy + sort.Float64s(valuesCopy) // Step 2: Calculate Q1 and Q3 - Q1 := values_copy[len(values_copy)/4] - Q3 := values_copy[3*len(values_copy)/4] + Q1 := valuesCopy[len(valuesCopy)/4] + Q3 := valuesCopy[3*len(valuesCopy)/4] // Step 3: Calculate IQR IQR := Q3 - Q1 @@ -47,7 +47,7 @@ func (s *Info) CalculateAvg() float64 { // Step 5: Filter out the outliers cnt := 0 avg := 0.0 - for _, value := range values_copy { + for _, value := range valuesCopy { if value >= lowerBound && value <= upperBound { if cnt == 0 { avg = value From eedeb7a5c91a65f22d37f80b67d4f84a7cf57822 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:53:14 -0700 Subject: [PATCH 55/88] fix lint --- test/engine/setup/monitor/statistic.go | 14 +++++++------- test/engine/trigger/file.go | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index 55412df129..d5e900a105 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -27,15 +27,15 @@ func (s *Info) CalculateMax() float64 { } func (s *Info) CalculateAvg() float64 { - values_copy := make([]float64, len(s.values)) - copy(values_copy, s.values) + valuesCopy := make([]float64, len(s.values)) + copy(valuesCopy, s.values) - // Step 1: Sort the values_copy - sort.Float64s(values_copy) + // Step 1: Sort the valuesCopy + sort.Float64s(valuesCopy) // Step 2: Calculate Q1 and Q3 - Q1 := values_copy[len(values_copy)/4] - Q3 := values_copy[3*len(values_copy)/4] + Q1 := valuesCopy[len(valuesCopy)/4] + Q3 := valuesCopy[3*len(valuesCopy)/4] // Step 3: Calculate IQR IQR := Q3 - Q1 @@ -47,7 +47,7 @@ func (s *Info) CalculateAvg() float64 { // Step 5: Filter out the outliers cnt := 0 avg := 0.0 - for _, value := range values_copy { + for _, value := range valuesCopy { if value >= lowerBound && value <= upperBound { if cnt == 0 { avg = value diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 83cfe2dfd3..9c94b98251 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -33,9 +33,9 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, } maxLen := 0 - for i := 0; i < len(JSONTemplates); i++ { - if len(JSONTemplates[i]) > maxLen { - maxLen = len(JSONTemplates[i]) + for j := 0; j < len(JSONTemplates); j++ { + if len(JSONTemplates[j]) > maxLen { + maxLen = len(JSONTemplates[j]) } } From e2df97b0062ebaa9e5ed839258c7c25ada16ddda Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:53:50 -0700 Subject: [PATCH 56/88] fix lint --- test/engine/trigger/file.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 83cfe2dfd3..9c94b98251 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -33,9 +33,9 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, } maxLen := 0 - for i := 0; i < len(JSONTemplates); i++ { - if len(JSONTemplates[i]) > maxLen { - maxLen = len(JSONTemplates[i]) + for j := 0; j < len(JSONTemplates); j++ { + if len(JSONTemplates[j]) > maxLen { + maxLen = len(JSONTemplates[j]) } } From b55203cf4779eb6b7c980c9e1c21f0567fc91e32 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 05:57:25 -0700 Subject: [PATCH 57/88] update file trigger --- test/engine/trigger/file.go | 91 ++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 9c94b98251..32d5a181b4 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -6,7 +6,6 @@ import ( "math/rand" "os" "path/filepath" - "sync" "time" "golang.org/x/time/rate" @@ -16,64 +15,50 @@ import ( // JSON template func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - numGoRoutines := (speed-1)/5 + 1 - if numGoRoutines > 8 { - numGoRoutines = 8 + JSONTemplates := []string{ + `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`, + `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`, + `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`, + `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`, + `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, + } + maxLen := 0 + for j := 0; j < len(JSONTemplates); j++ { + if len(JSONTemplates[j]) > maxLen { + maxLen = len(JSONTemplates[j]) + } } - wg := sync.WaitGroup{} - for i := 0; i < numGoRoutines; i++ { - wg.Add(1) - go func(ctx context.Context, thread_id, speed, totalTime int, path string) { - defer wg.Done() - JSONTemplates := []string{ - `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`, - `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`, - `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`, - `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`, - `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, - } - maxLen := 0 - for j := 0; j < len(JSONTemplates); j++ { - if len(JSONTemplates[j]) > maxLen { - maxLen = len(JSONTemplates[j]) - } - } - // clear file - path = filepath.Clean(path) - path = fmt.Sprintf("%d_%s", thread_id, path) - path = filepath.Join(config.CaseHome, path) - fmt.Println(path) - _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 + // clear file + path = filepath.Clean(path) + path = filepath.Join(config.CaseHome, path) + fmt.Println(path) + _ = os.WriteFile(path, []byte{}, 0600) + file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) - timeout := time.After(time.Minute * time.Duration(totalTime)) + timeout := time.After(time.Minute * time.Duration(totalTime)) - rand.Seed(time.Now().UnixNano()) + rand.Seed(time.Now().UnixNano()) - for { - select { - // context is done - case <-ctx.Done(): - // clear file - _ = file.Close() - return - // all time is done - case <-timeout: - // clear file - _ = file.Close() - return - default: - if limiter.AllowN(time.Now(), maxLen) { - randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 - _, _ = file.WriteString(JSONTemplates[randomIndex]) - } - } + for { + select { + // context is done + case <-ctx.Done(): + // clear file + _ = file.Close() + return ctx, nil + // all time is done + case <-timeout: + // clear file + _ = file.Close() + return ctx, nil + default: + if limiter.AllowN(time.Now(), maxLen) { + randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 + _, _ = file.WriteString(JSONTemplates[randomIndex]) } - }(ctx, i, speed/numGoRoutines, totalTime, path) + } } - wg.Wait() - return ctx, nil } From 78e2165f933c120328ed0d92f9323d6e6352fe86 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 06:30:52 -0700 Subject: [PATCH 58/88] update feature --- .../performance_file_to_blackhole_filebeat/case.feature | 7 ------- .../performance_file_to_blackhole_fluentbit/case.feature | 7 ------- .../performance_file_to_blackhole_ilogtail/case.feature | 7 ------- .../performance_file_to_blackhole_vector/case.feature | 7 ------- 4 files changed, 28 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 1a24a6858f..8d5e5fc99c 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 9358e5311d..0d6c72174f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index b531833756..6ba5f72d22 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 12cc1d2060..9ccf02bb4e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark From 86e2ad18450be1309ad61f8bb9685d0db25ffa47 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Fri, 18 Oct 2024 06:32:25 -0700 Subject: [PATCH 59/88] update case.feature --- .../performance_file_to_blackhole_filebeat/case.feature | 7 ------- .../performance_file_to_blackhole_fluentbit/case.feature | 7 ------- .../performance_file_to_blackhole_ilogtail/case.feature | 7 ------- .../performance_file_to_blackhole_vector/case.feature | 7 ------- 4 files changed, 28 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index 1a24a6858f..8d5e5fc99c 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 9358e5311d..0d6c72174f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index b531833756..6ba5f72d22 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 12cc1d2060..9ccf02bb4e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,12 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} -<<<<<<< HEAD - When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ -======= When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished ->>>>>>> benchmark From 0afc9ec8a7fff6e4efc28e7bfa63b1fbd31107cd Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Sun, 20 Oct 2024 18:15:17 -0700 Subject: [PATCH 60/88] update benchmark.md --- docs/cn/developer-guide/test/benchmark.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index d73ff101ac..ed1a59ef93 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -29,10 +29,8 @@ Feature: performance file to blackhole vector Given docker-compose type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When wait monitor until log processing finished ``` - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 @@ -40,7 +38,8 @@ Feature: performance file to blackhole vector - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 - `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 -- `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) +- `When generate random logs to file`: 向文件中按照速率生成json格式测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) +- `When wait monitor util log processing finished`: 等待采集agent日志收集完成 ### 运行测试 From e5098744c61242eb46ab46770644fc8106c461c1 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 00:27:27 -0700 Subject: [PATCH 61/88] update trigger to nginx logs --- docs/cn/developer-guide/test/benchmark.md | 8 +- .../docker-compose.yaml | 10 --- .../filebeat.yml | 19 ----- .../main.conf | 4 +- .../parsers.conf | 5 +- .../docker-compose.yaml | 2 +- .../ilogtail.yaml | 17 ++++- .../case.feature | 10 +-- .../docker-compose.yaml | 26 +++++++ .../ilogtail.yaml | 21 +++++ .../vector.yaml | 8 +- test/engine/setup/monitor/monitor.go | 31 +++++--- test/engine/trigger/file.go | 76 ++++++++++++++----- 13 files changed, 161 insertions(+), 76 deletions(-) delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml rename test/benchmark/test_cases/{performance_file_to_blackhole_filebeat => performance_file_to_blackhole_ilogtailspl}/case.feature (52%) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index d73ff101ac..992ac80661 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -29,10 +29,8 @@ Feature: performance file to blackhole vector Given docker-compose type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template - """ - {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} - """ + When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When wait monitor until log processing finished ``` - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 @@ -40,7 +38,7 @@ Feature: performance file to blackhole vector - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 - `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 -- `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) +- `When generate random json logs to file`: 向文件中按照固定速率随机生成json格式测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) ### 运行测试 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml deleted file mode 100644 index ce6b83c442..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3.8' - -services: - filebeat: - image: docker.elastic.co/beats/filebeat:8.14.2 - user: root - volumes: - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - .:/home/filebeat - command: filebeat -e --strict.perms=false diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml deleted file mode 100644 index e93a2688e0..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml +++ /dev/null @@ -1,19 +0,0 @@ -filebeat.inputs: -- type: filestream - id: input-file - paths: - - /home/filebeat/*.log - prospector.scanner.check_interval: 1s - -processors: - - decode_json_fields: - fields: ["message"] - target: "json" - - drop_event: - when: - not: - equals: - json.user-agent: "no-agent" - -output.console: - pretty: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf index d009876b60..307b6d1d04 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -4,13 +4,13 @@ [INPUT] name tail path /home/fluentbit/*.log - parser json + parser nginx refresh_interval 1 [FILTER] name grep match * - regex user-agent no-agent + regex user_agent no-agent [OUTPUT] name stdout diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf index 9229d0ab6a..387b51f0cc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf @@ -1,3 +1,4 @@ [PARSER] - Name json - Format json \ No newline at end of file + Name nginx + Format regex + Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)") \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index f5f31279dc..97bf5deb2e 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml index 3bde206ddf..a346a84261 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/ilogtail.yaml @@ -4,11 +4,24 @@ inputs: FilePaths: - /home/ilogtail/*.log processors: - - Type: processor_parse_json_native + - Type: processor_parse_regex_native SourceKey: content + Regex: (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*) (.*) \[(\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4})\] \"(\w+) (.*?) HTTP\/(\d+\.\d+)\" (\d{3}) (\d+) \"(.*?)\" \"(.*?)\" + Keys: + - ip + - ident + - auth + - timestamp + - method + - request + - http_version + - response_code + - bytes + - referrer + - user_agent - Type: processor_filter_regex_native FilterKey: - - user-agent + - user_agent FilterRegex: - ^no-agent$ flushers: diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature similarity index 52% rename from test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature rename to test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature index 8d5e5fc99c..6e4b374f97 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature @@ -1,12 +1,12 @@ @input -Feature: performance file to blackhole filebeat - Performance file to blackhole filebeat +Feature: performance file to blackhole iLogtail + Performance file to blackhole iLogtail @e2e-performance @docker-compose - Scenario: PerformanceFileToBlackholeFilebeat + Scenario: PerformanceFileToBlackholeiLogtailSPL Given {docker-compose} environment Given docker-compose boot type {benchmark} - When start docker-compose {performance_file_to_blackhole_filebeat} - When start monitor {filebeat} + When start docker-compose {performance_file_to_blackhole_ilogtailspl} + When start monitor {ilogtailC} When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml new file mode 100644 index 0000000000..97bf5deb2e --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml @@ -0,0 +1,26 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + ilogtailC: + image: aliyun/loongcollector:0.0.1 + volumes: + - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - .:/home/ilogtail + healthcheck: + test: "cat /loongcollector/log/loongcollector.LOG" + interval: 15s + timeout: 5s diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml new file mode 100644 index 0000000000..28b3945785 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml @@ -0,0 +1,21 @@ +enable: true +inputs: + - Type: input_file + FilePaths: + - /home/ilogtail/*.log +processors: + - Type: processor_spl + Script: | + * | parse-regexp content, '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*) (.*) \[(\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4})\] \"(\w+) (.*?) HTTP\/(\d+\.\d+)\" (\d{3}) (\d+) \"(.*?)\" \"(.*?)\"' as ip, ident, auth, timestamp, method, request, http_version, response_code, bytes, referrer, user_agent + - Type: processor_filter_regex_native + FilterKey: + - user_agent + FilterRegex: + - ^no-agent$ +flushers: + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore + diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml index 20bb85b721..48dceff3a2 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml @@ -5,19 +5,19 @@ sources: - /home/vector-log/*.log transforms: - transform_json: + transform_regex: type: remap inputs: - input_file source: |- - . = parse_json!(string!(.message)) + . = parse_regex!(.message, r'^(?P[^ ]*) (?P[^ ]*) (?P[^ ]*) \[(?P[^\]]*)\] "(?P\S+)(?: +(?P[^\"]*?)(?: +\S*)?)?" (?P[^ ]*) (?P[^ ]*)(?: "(?P[^\"]*)" "(?P[^\"]*)")') filter_agent: type: filter inputs: - - transform_json + - transform_regex condition: type: "vrl" - source: ."user-agent" == "no-agent" + source: ."user_agent" == "no-agent" sinks: output_std: diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 740062367a..efa66023f2 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -21,12 +21,17 @@ const ( interval = 3 ) +const ( + exitCodeSuccess = iota + exitCodeErrorNotProcessing +) + var monitor Monitor type Monitor struct { isMonitoring atomic.Bool statistic *Statistic - stopCh chan struct{} + stopCh chan int } func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { @@ -34,8 +39,15 @@ func StartMonitor(ctx context.Context, containerName string) (context.Context, e } func WaitMonitorUntilProcessingFinished(ctx context.Context) (context.Context, error) { - <-monitor.Done() - return ctx, nil + errCode := <-monitor.Done() + switch errCode { + case exitCodeSuccess: + return ctx, nil + case exitCodeErrorNotProcessing: + return ctx, fmt.Errorf("monitoring error: CPU usage is too low, not processing") + default: + return ctx, fmt.Errorf("monitoring error: unknown error") + } } func (m *Monitor) Start(ctx context.Context, containerName string) (context.Context, error) { @@ -55,7 +67,7 @@ func (m *Monitor) Start(ctx context.Context, containerName string) (context.Cont if strings.Contains(containerFullName, containerName) { m.isMonitoring.Store(true) m.statistic = NewMonitorStatistic(config.CaseName) - m.stopCh = make(chan struct{}) + m.stopCh = make(chan int) fmt.Println("Start monitoring container:", containerFullName) go m.monitoring(client, containerFullName) return ctx, nil @@ -72,7 +84,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { statisticFile := reportDir + config.CaseName + "_statistic.json" recordsFile := reportDir + config.CaseName + "_records.json" // calculate low threshold after 60 seconds - timer := time.NewTimer(60 * time.Second) + timerCal := time.NewTimer(60 * time.Second) lowThreshold := 0.0 outlierCnt := 0 // read from cadvisor per interval seconds @@ -81,7 +93,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { request := &v1.ContainerInfoRequest{NumStats: 10} for { select { - case <-timer.C: + case <-timerCal.C: // 计算CPU使用率的下阈值 cpuRawData := make([]float64, len(m.statistic.GetCPURawData())-5) copy(cpuRawData, m.statistic.GetCPURawData()[5:]) @@ -95,7 +107,8 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { fmt.Println("3/4 of CPU usage rate(%):", cpuRawData[3*len(cpuRawData)/4]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) if lowThreshold < 0 { - m.stopCh <- struct{}{} + m.stopCh <- exitCodeErrorNotProcessing + return } case <-ticker.C: // 获取容器信息 @@ -117,7 +130,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { bytes, _ = m.statistic.MarshalRecordsJSON() _ = os.WriteFile(recordsFile, bytes, 0600) m.isMonitoring.Store(false) - m.stopCh <- struct{}{} + m.stopCh <- exitCodeSuccess m.statistic.ClearStatistic() return } @@ -125,6 +138,6 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { } } -func (m *Monitor) Done() <-chan struct{} { +func (m *Monitor) Done() <-chan int { return m.stopCh } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 32d5a181b4..d7cf83ede8 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -15,19 +15,6 @@ import ( // JSON template func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - JSONTemplates := []string{ - `{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`, - `{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`, - `{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`, - `{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`, - `{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`, - } - maxLen := 0 - for j := 0; j < len(JSONTemplates); j++ { - if len(JSONTemplates[j]) > maxLen { - maxLen = len(JSONTemplates[j]) - } - } // clear file path = filepath.Clean(path) @@ -36,7 +23,10 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path _ = os.WriteFile(path, []byte{}, 0600) file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen) + maxLogLen := 1024 + nginxLog := genNginxLog() + + limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLogLen) timeout := time.After(time.Minute * time.Duration(totalTime)) @@ -55,10 +45,62 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path _ = file.Close() return ctx, nil default: - if limiter.AllowN(time.Now(), maxLen) { - randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404 - _, _ = file.WriteString(JSONTemplates[randomIndex]) + if limiter.AllowN(time.Now(), len(nginxLog)) { + _, _ = file.WriteString(nginxLog + "\n") // #nosec G307 + nginxLog = genNginxLog() } } } } + +var ipAddresses = []string{ + "103.159.151.180", + "12.55.18.241", + "182.233.128.102", + "221.85.57.231", + "76.245.65.224", + "86.250.231.93", + "44.201.253.252", + "218.7.2.219", + "172.118.174.109", + "208.16.46.154", + "7.138.80.41", + "214.73.25.80", + "83.124.20.79", + "80.226.48.153", + "92.129.204.161", + "212.103.145.159", + "148.188.8.90", + "148.212.244.121", + "106.186.172.157", + "30.127.196.158", +} + +var userAgents = []string{ + "aliyun-sdk-java", + "aliyun-sdk-golang", + "aliyun-sdk-python", +} + +var statusCodes = []string{ + "400", + "401", + "402", + "403", + "404", + "200", +} + +func genNginxLog() string { // nosec G404 + nginxLogTemplate := `%s - - [%s] "DELETE http://www.districtdot-com.biz/syndicate HTTP/1.1" %s 3715 "http://www.chiefscalable.biz/webservices" "%s"` + currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") + ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] + statusIdx := rand.Intn(len(statusCodes) * 10) + if statusIdx >= len(statusCodes) { + statusIdx = len(statusCodes) - 1 + } + statusCode := statusCodes[statusIdx] + userAgent := userAgents[rand.Intn(len(userAgents))] + + return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, userAgent) +} From 2bdf8fe5056eaebd72f417834a4c4c52e3dd16e9 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 00:48:12 -0700 Subject: [PATCH 62/88] polish --- .github/workflows/benchmark.yaml | 3 +++ test/engine/trigger/file.go | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 8b3b40c7fe..7ae82afc0a 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -28,6 +28,9 @@ on: - 'CHANGELOG.md' # types: # - closed + push: + branched: + - benchmark jobs: CI: diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index d7cf83ede8..b5749d4e12 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -23,6 +23,7 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path _ = os.WriteFile(path, []byte{}, 0600) file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 + rand.Seed(time.Now().UnixNano()) maxLogLen := 1024 nginxLog := genNginxLog() @@ -30,8 +31,6 @@ func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path timeout := time.After(time.Minute * time.Duration(totalTime)) - rand.Seed(time.Now().UnixNano()) - for { select { // context is done @@ -91,16 +90,16 @@ var statusCodes = []string{ "200", } -func genNginxLog() string { // nosec G404 +func genNginxLog() string { nginxLogTemplate := `%s - - [%s] "DELETE http://www.districtdot-com.biz/syndicate HTTP/1.1" %s 3715 "http://www.chiefscalable.biz/webservices" "%s"` currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") - ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] - statusIdx := rand.Intn(len(statusCodes) * 10) + ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // nosec G404 + statusIdx := rand.Intn(len(statusCodes) * 10) // nosec G404 if statusIdx >= len(statusCodes) { statusIdx = len(statusCodes) - 1 } statusCode := statusCodes[statusIdx] - userAgent := userAgents[rand.Intn(len(userAgents))] + userAgent := userAgents[rand.Intn(len(userAgents))] // nosec G404 return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, userAgent) } From 0cd228f9720b0659263ce9c4279e0b6357e3ea1e Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 01:26:23 -0700 Subject: [PATCH 63/88] polish --- .../performance_file_to_blackhole_ilogtail/docker-compose.yaml | 2 +- .../docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 97bf5deb2e..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml index 97bf5deb2e..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" From 3623c97fba672b1c97b38d2b10608baa89cdd8df Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 05:15:16 -0700 Subject: [PATCH 64/88] polish --- .../case.feature | 2 +- .../case.feature | 2 +- .../docker-compose.yaml | 26 ------------------- .../case.feature | 2 +- .../docker-compose.yaml | 3 ++- .../ilogtail.yaml | 14 +++------- .../case.feature | 2 +- test/engine/steps.go | 2 +- test/engine/trigger/file.go | 8 +++--- 9 files changed, 15 insertions(+), 46 deletions(-) delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 0d6c72174f..96d7fa9b36 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 6ba5f72d22..08ab9a205a 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml deleted file mode 100644 index f5f31279dc..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '3.8' - -services: - ilogtailC: - image: aliyun/loongcollector:0.0.1 - volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - - .:/home/ilogtail - healthcheck: - test: "cat /loongcollector/log/loongcollector.LOG" - interval: 15s - timeout: 5s diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature index 6e4b374f97..03731669a4 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtailspl} When start monitor {ilogtailC} - When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml index f5f31279dc..4bd6183793 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml @@ -18,8 +18,9 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail + - ../log:/usr/local/loongcollector/log healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" interval: 15s diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml index 28b3945785..6f0050c372 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml @@ -7,15 +7,9 @@ processors: - Type: processor_spl Script: | * | parse-regexp content, '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*) (.*) \[(\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4})\] \"(\w+) (.*?) HTTP\/(\d+\.\d+)\" (\d{3}) (\d+) \"(.*?)\" \"(.*?)\"' as ip, ident, auth, timestamp, method, request, http_version, response_code, bytes, referrer, user_agent - - Type: processor_filter_regex_native - FilterKey: - - user_agent - FilterRegex: - - ^no-agent$ + | where user_agent='no-agent' flushers: - - Type: flusher_sls - Region: cn-hangzhou - Endpoint: cn-hangzhou.log.aliyuncs.com - Project: test_project - Logstore: test_logstore + - Type: flusher_stdout + OnlyStdout: true + Tags: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index 9ccf02bb4e..bc3fd8d10f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/engine/steps.go b/test/engine/steps.go index cc73b459ba..b3ec512458 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -42,7 +42,7 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^query through \{(.*)\}`, control.SetQuery) ctx.When(`^begin trigger`, trigger.BeginTrigger) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) - ctx.When(`^generate random json logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomJSONLogToFile) + ctx.When(`^generate random nginx logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, trigger.GenerateRandomNginxLogToFile) ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) ctx.When(`^wait monitor until log processing finished$`, monitor.WaitMonitorUntilProcessingFinished) diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index b5749d4e12..0d603f3400 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -14,7 +14,7 @@ import ( ) // JSON template -func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { +func GenerateRandomNginxLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { // clear file path = filepath.Clean(path) @@ -93,13 +93,13 @@ var statusCodes = []string{ func genNginxLog() string { nginxLogTemplate := `%s - - [%s] "DELETE http://www.districtdot-com.biz/syndicate HTTP/1.1" %s 3715 "http://www.chiefscalable.biz/webservices" "%s"` currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") - ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // nosec G404 - statusIdx := rand.Intn(len(statusCodes) * 10) // nosec G404 + ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // #nosec G404 + statusIdx := rand.Intn(len(statusCodes) * 10) // #nosec G404 if statusIdx >= len(statusCodes) { statusIdx = len(statusCodes) - 1 } statusCode := statusCodes[statusIdx] - userAgent := userAgents[rand.Intn(len(userAgents))] // nosec G404 + userAgent := userAgents[rand.Intn(len(userAgents))] // #nosec G404 return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, userAgent) } From 94e5e5bf5d84b7a9113a376bd785fb1d8c7ca63e Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 07:47:16 -0700 Subject: [PATCH 65/88] update ilogtailspl --- .../docker-compose.yaml | 26 +++++++++++++++++++ .../docker-compose.yaml | 3 +-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml new file mode 100644 index 0000000000..f5f31279dc --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -0,0 +1,26 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + ilogtailC: + image: aliyun/loongcollector:0.0.1 + volumes: + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - .:/home/ilogtail + healthcheck: + test: "cat /loongcollector/log/loongcollector.LOG" + interval: 15s + timeout: 5s diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml index 4bd6183793..f5f31279dc 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml @@ -18,9 +18,8 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/usr/local/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml - .:/home/ilogtail - - ../log:/usr/local/loongcollector/log healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" interval: 15s From a75e5383a096333771aa3b7d4f915085e72e7c73 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 18:23:21 -0700 Subject: [PATCH 66/88] empty commit From d938bc22c615bf98eeb360e556140466f5d08e70 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 18:51:49 -0700 Subject: [PATCH 67/88] fix bug --- test/engine/setup/monitor/monitor.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index efa66023f2..128ea21fd5 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -24,6 +24,7 @@ const ( const ( exitCodeSuccess = iota exitCodeErrorNotProcessing + exitCodeErrorGetContainerInfo ) var monitor Monitor @@ -45,6 +46,8 @@ func WaitMonitorUntilProcessingFinished(ctx context.Context) (context.Context, e return ctx, nil case exitCodeErrorNotProcessing: return ctx, fmt.Errorf("monitoring error: CPU usage is too low, not processing") + case exitCodeErrorGetContainerInfo: + return ctx, fmt.Errorf("monitoring error: failed to get container info, maybe container crashed") default: return ctx, fmt.Errorf("monitoring error: unknown error") } @@ -107,14 +110,18 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { fmt.Println("3/4 of CPU usage rate(%):", cpuRawData[3*len(cpuRawData)/4]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) if lowThreshold < 0 { + m.isMonitoring.Store(false) m.stopCh <- exitCodeErrorNotProcessing + m.statistic.ClearStatistic() return } case <-ticker.C: // 获取容器信息 containerInfo, err := client.DockerContainer(containerName, request) if err != nil { - fmt.Println("Error getting container info:", err) + m.isMonitoring.Store(false) + m.stopCh <- exitCodeErrorGetContainerInfo + m.statistic.ClearStatistic() return } for _, stat := range containerInfo.Stats { From 62ce72f0c64dc85659663a0d30e66a7616552287 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 20:08:24 -0700 Subject: [PATCH 68/88] update spl ilogtail.yaml --- .../ilogtail.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml index 6f0050c372..99cf637fe2 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/ilogtail.yaml @@ -9,7 +9,9 @@ processors: * | parse-regexp content, '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*) (.*) \[(\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4})\] \"(\w+) (.*?) HTTP\/(\d+\.\d+)\" (\d{3}) (\d+) \"(.*?)\" \"(.*?)\"' as ip, ident, auth, timestamp, method, request, http_version, response_code, bytes, referrer, user_agent | where user_agent='no-agent' flushers: - - Type: flusher_stdout - OnlyStdout: true - Tags: true + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore From c16c26212bed6cb5fdd9ad1546180cbd3a4d30f5 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 20:09:37 -0700 Subject: [PATCH 69/88] update docs --- docs/cn/developer-guide/test/benchmark.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md index 52a15a2a58..ed1a59ef93 100644 --- a/docs/cn/developer-guide/test/benchmark.md +++ b/docs/cn/developer-guide/test/benchmark.md @@ -38,12 +38,8 @@ Feature: performance file to blackhole vector - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 - `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 -<<<<<<< HEAD - `When generate random logs to file`: 向文件中按照速率生成json格式测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) - `When wait monitor util log processing finished`: 等待采集agent日志收集完成 -======= -- `When generate random json logs to file`: 向文件中按照固定速率随机生成json格式测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) ->>>>>>> e2e-performance-test ### 运行测试 From cdb2c26083ff0532c0d1e745fce3c249da8e471c Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Mon, 21 Oct 2024 22:27:04 -0700 Subject: [PATCH 70/88] update github action benchmark.yaml --- .github/workflows/benchmark.yaml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 7ae82afc0a..bb1491be19 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,15 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - # types: - # - closed - push: - branched: - - benchmark + types: + - closed jobs: CI: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -105,12 +102,12 @@ jobs: last_commit_message=$(git log -1 --pretty=%B) git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # Push gh-pages branch by yourself - - name: Push benchmark result - run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages + # TODO: Push gh-pages branch by yourself + # - name: Push benchmark result + # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages result: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] From 3141b111334fe0a1fc2a0df0112d794dde481dee Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 22 Oct 2024 18:00:19 -0700 Subject: [PATCH 71/88] update trigger file --- .github/workflows/benchmark.yaml | 2 +- test/engine/trigger/file.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index bb1491be19..30699f0761 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -102,7 +102,7 @@ jobs: last_commit_message=$(git log -1 --pretty=%B) git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" - # TODO: Push gh-pages branch by yourself + # TODO: Push gh-pages branch by github-action-bot # - name: Push benchmark result # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 0d603f3400..373aebbe52 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -90,16 +90,25 @@ var statusCodes = []string{ "200", } +const bytesMean = 5500.0 +const bytesStddev = 1500.0 + func genNginxLog() string { - nginxLogTemplate := `%s - - [%s] "DELETE http://www.districtdot-com.biz/syndicate HTTP/1.1" %s 3715 "http://www.chiefscalable.biz/webservices" "%s"` + nginxLogTemplate := `%s - - [%s] "GET http://www.districtdot-com.biz/syndicate HTTP/1.1" %s %d "http://www.chiefscalable.biz/webservices" "%s"` currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // #nosec G404 statusIdx := rand.Intn(len(statusCodes) * 10) // #nosec G404 if statusIdx >= len(statusCodes) { statusIdx = len(statusCodes) - 1 } + bytesSize := int32(rand.NormFloat64()*bytesStddev + bytesMean) + if bytesSize < 1000 { + bytesSize = 0 + } else if bytesSize > 10000 { + bytesSize = 10000 + } statusCode := statusCodes[statusIdx] userAgent := userAgents[rand.Intn(len(userAgents))] // #nosec G404 - return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, userAgent) + return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, bytesSize, userAgent) } From dc9c3925b6f35041bb290b93684859381235aa41 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Tue, 22 Oct 2024 22:37:14 -0700 Subject: [PATCH 72/88] add testcases filebeat, logstash --- .../case.feature | 12 +++++++++++ .../docker-compose.yaml | 10 ++++++++++ .../filebeat.yml | 20 +++++++++++++++++++ .../case.feature | 12 +++++++++++ .../docker-compose.yaml | 12 +++++++++++ .../logstash.conf | 15 ++++++++++++++ 6 files changed, 81 insertions(+) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature new file mode 100644 index 0000000000..ad35f0ec00 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -0,0 +1,12 @@ +@input +Feature: performance file to blackhole filebeat + Performance file to blackhole filebeat + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeFilebeat + Given {docker-compose} environment + Given docker-compose boot type {benchmark} + When start docker-compose {performance_file_to_blackhole_filebeat} + When start monitor {filebeat} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml new file mode 100644 index 0000000000..ce6b83c442 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -0,0 +1,10 @@ +version: '3.8' + +services: + filebeat: + image: docker.elastic.co/beats/filebeat:8.14.2 + user: root + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - .:/home/filebeat + command: filebeat -e --strict.perms=false diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml new file mode 100644 index 0000000000..b4ad00ac22 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -0,0 +1,20 @@ +filebeat.inputs: +- type: filestream + id: input-file + paths: + - /home/filebeat/*.log + prospector.scanner.check_interval: 1s + +processors: + - dissect: + tokenizer: '%{ip} %{ident} %{auth} [%{timestamp}] "%{method} %{request} HTTP/%{http_version}" %{response_code} %{bytes} "%{referrer}" "%{user_agent}"' + field: "message" + target_prefix: "parsed" + - drop_event: + when: + not: + equals: + parsed.user_agent: "no-agent" + +output.console: + pretty: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature new file mode 100644 index 0000000000..96eef43009 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature @@ -0,0 +1,12 @@ +@input +Feature: performance file to blackhole logstash + Performance file to blackhole logstash + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeLogstash + Given {docker-compose} environment + Given docker-compose boot type {benchmark} + When start docker-compose {performance_file_to_blackhole_logstash} + When start monitor {logstash} + When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml new file mode 100644 index 0000000000..54f9d88678 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml @@ -0,0 +1,12 @@ +version: '3.8' + +services: + + logstash: + image: docker.elastic.co/logstash/logstash:8.15.3 + volumes: + - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf + - .:/home/logstash + restart: always + + diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf new file mode 100644 index 0000000000..a006b5b2c3 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf @@ -0,0 +1,15 @@ +input { + file { + path => "/home/logstash/*.log" + start_position => "beginning" + } +} +filter { + grok { + match => { "message" => "%{IP:ip} %{DATA:ident} %{DATA:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{DATA:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} (?:%{NUMBER:bytes}|-) \"%{DATA:referrer}\" \"%{DATA:user_agent}\"" } + } + drop {} +} +output { + stdout {} +} \ No newline at end of file From 42d42ab25dcbd4e37366ae5e5b1c9a4773b4bfa7 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 23 Oct 2024 01:01:28 -0700 Subject: [PATCH 73/88] polish --- plugins/input/prometheus/input_prometheus.go | 190 ------------------- test/engine/setup/monitor/monitor.go | 7 +- 2 files changed, 3 insertions(+), 194 deletions(-) delete mode 100644 plugins/input/prometheus/input_prometheus.go diff --git a/plugins/input/prometheus/input_prometheus.go b/plugins/input/prometheus/input_prometheus.go deleted file mode 100644 index 0ecd38100c..0000000000 --- a/plugins/input/prometheus/input_prometheus.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2021 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prometheus - -import ( - "errors" - "flag" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" - liblogger "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - - "github.com/alibaba/ilogtail/pkg/config" - "github.com/alibaba/ilogtail/pkg/logger" - "github.com/alibaba/ilogtail/pkg/pipeline" -) - -var libLoggerOnce sync.Once - -type ServiceStaticPrometheus struct { - Yaml string `comment:"the prometheus configuration content, more details please see [here](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)"` - ConfigFilePath string `comment:"the prometheus configuration path, and the param would be ignored when Yaml param is configured."` - AuthorizationPath string `comment:"the prometheus authorization path, only using in authorization files. When Yaml param is configured, the default value is the current binary path. However, the default value is the ConfigFilePath directory when ConfigFilePath is working."` - ExtraFlags map[string]string `comment:"the prometheus extra configuration flags, like promscrape.maxScrapeSize, for more flags please see [here](https://docs.victoriametrics.com/vmagent.html#advanced-usage)"` - NoStaleMarkers bool `comment:"Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series"` - - scraper *promscrape.Scraper //nolint:typecheck - shutdown chan struct{} - waitGroup sync.WaitGroup - context pipeline.Context - lock sync.Mutex - running bool - collector pipeline.Collector - kubeMeta *KubernetesMeta -} - -func (p *ServiceStaticPrometheus) Init(context pipeline.Context) (int, error) { - p.kubeMeta = NewKubernetesMeta(context) - if p.kubeMeta.readKubernetesWorkloadMeta() { - promscrape.ConfigMemberInfo(int(p.kubeMeta.replicas), strconv.Itoa(p.kubeMeta.currentNum)) - } - libLoggerOnce.Do(func() { - if f := flag.Lookup("loggerOutput"); f != nil { - _ = f.Value.Set("stdout") - } - // set max scrape size to 256MB - err := flag.Set("promscrape.maxScrapeSize", "268435456") - logger.Info(context.GetRuntimeContext(), "set config maxScrapeSize to 256MB, error", err) - liblogger.Init() - common.StartUnmarshalWorkers() - if p.NoStaleMarkers { - err := flag.Set("promscrape.noStaleMarkers", "true") - logger.Info(context.GetRuntimeContext(), "set config", "promscrape.noStaleMarkers", "value", "true", "error", err) - } - for k, v := range p.ExtraFlags { - err := flag.Set(k, v) - logger.Info(context.GetRuntimeContext(), "set config", k, "value", v, "error", err) - } - }) - p.context = context - var detail []byte - switch { - case p.Yaml != "": - detail = []byte(p.Yaml) - if p.AuthorizationPath == "" { - p.AuthorizationPath = config.LoongcollectorGlobalConfig.LoongcollectorConfDir - } - case p.ConfigFilePath != "": - f, err := os.Open(p.ConfigFilePath) - if err != nil { - return 0, fmt.Errorf("cannot find prometheus configuration file") - } - defer func(f *os.File) { - _ = f.Close() - }(f) - bytes, err := io.ReadAll(f) - if err != nil { - return 0, fmt.Errorf("cannot read prometheus configuration file") - } - detail = bytes - if p.AuthorizationPath == "" { - p.AuthorizationPath = filepath.Dir(p.ConfigFilePath) - } - default: - return 0, errors.New("the scrape configuration is required") - } - var err error - if p.AuthorizationPath, err = filepath.Abs(p.AuthorizationPath); err != nil { - return 0, fmt.Errorf("cannot find the abs authorization path: %v", err) - } - name := strings.Join([]string{context.GetProject(), context.GetLogstore(), context.GetConfigName()}, "_") - p.scraper = promscrape.NewScraper(detail, name, p.AuthorizationPath) //nolint:typecheck - if err := p.scraper.CheckConfig(); err != nil { - return 0, fmt.Errorf("illegal prometheus configuration file %s: %v", name, err) - } - return 0, nil -} - -func (p *ServiceStaticPrometheus) Description() string { - return "prometheus scrape plugin for logtail, use vmagent lib" -} - -// Start starts the ServiceInput's service, whatever that may be -func (p *ServiceStaticPrometheus) Start(c pipeline.Collector) error { - p.collector = c - p.shutdown = make(chan struct{}) - p.waitGroup.Add(1) - defer p.waitGroup.Done() - p.scraper.Init(p.slsPushData) - p.running = true - if p.kubeMeta.isWorkingOnClusterMode() { - p.StartKubeReloadScraper() - } - <-p.shutdown - p.scraper.Stop() - return nil -} - -// Stop stops the services and closes any necessary channels and connections -func (p *ServiceStaticPrometheus) Stop() error { - p.lock.Lock() - defer p.lock.Unlock() - p.running = false - close(p.shutdown) - p.waitGroup.Wait() - return nil -} - -func (p *ServiceStaticPrometheus) StartKubeReloadScraper() { - go func() { - ticker := time.NewTicker(time.Second * 10) - for { - select { - case <-p.shutdown: - return - case <-ticker.C: - change, err := p.kubeMeta.getPrometheusReplicas() - if !change || err != nil { - continue - } - logger.Info(p.context.GetRuntimeContext(), "found change replicas, would start reload prometheus scraper", p.kubeMeta.replicas) - p.lock.Lock() - if !p.running { - return - } - promscrape.ConfigMemberInfo(int(p.kubeMeta.replicas), strconv.Itoa(p.kubeMeta.currentNum)) - p.scraper.Stop() - p.scraper.Init(p.slsPushData) - p.lock.Unlock() - logger.Info(p.context.GetRuntimeContext(), "reload prometheus scraper done") - } - } - }() -} - -func init() { - pipeline.ServiceInputs["service_prometheus"] = func() pipeline.ServiceInput { - return &ServiceStaticPrometheus{ - NoStaleMarkers: true, - } - } -} - -func (p *ServiceStaticPrometheus) slsPushData(_ *auth.Token, wr *prompbmarshal.WriteRequest) { - logger.Debug(p.context.GetRuntimeContext(), "append new metrics", wr.Size()) - appendTSDataToSlsLog(p.collector, wr) - logger.Debug(p.context.GetRuntimeContext(), "append done", wr.Size()) -} diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 128ea21fd5..111c3cd85c 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -98,16 +98,15 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { select { case <-timerCal.C: // 计算CPU使用率的下阈值 - cpuRawData := make([]float64, len(m.statistic.GetCPURawData())-5) - copy(cpuRawData, m.statistic.GetCPURawData()[5:]) + cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) + copy(cpuRawData, m.statistic.GetCPURawData()) sort.Float64s(cpuRawData) + cpuRawData = cpuRawData[len(cpuRawData)/4:] Q1 := cpuRawData[len(cpuRawData)/4] Q3 := cpuRawData[3*len(cpuRawData)/4] IQR := Q3 - Q1 lowThreshold = Q1 - 1.5*IQR fmt.Println("median of CPU usage rate(%):", cpuRawData[len(cpuRawData)/2]) - fmt.Println("1/4 of CPU usage rate(%):", cpuRawData[len(cpuRawData)/4]) - fmt.Println("3/4 of CPU usage rate(%):", cpuRawData[3*len(cpuRawData)/4]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) if lowThreshold < 0 { m.isMonitoring.Store(false) From 000f3a7d80dcab973301fe5babec4165bbf374f7 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 23 Oct 2024 22:49:14 -0700 Subject: [PATCH 74/88] resolve conflict --- .github/workflows/benchmark.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 30699f0761..99756888fe 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,12 +26,15 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - types: - - closed + # types: + # - closed + push: + branches: + - benchmark jobs: CI: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -103,11 +106,12 @@ jobs: git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" # TODO: Push gh-pages branch by github-action-bot - # - name: Push benchmark result - # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + - name: Push benchmark result + # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages + run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages result: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] From 62aaf064ba0cbcb5547ec449a516c0a8424bbe9e Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Wed, 23 Oct 2024 23:46:46 -0700 Subject: [PATCH 75/88] update monitor quit logic --- test/engine/setup/monitor/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 111c3cd85c..09bf9595f7 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -101,7 +101,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) copy(cpuRawData, m.statistic.GetCPURawData()) sort.Float64s(cpuRawData) - cpuRawData = cpuRawData[len(cpuRawData)/4:] + cpuRawData = cpuRawData[len(cpuRawData)/4 : 3*len(cpuRawData)/4+1] Q1 := cpuRawData[len(cpuRawData)/4] Q3 := cpuRawData[3*len(cpuRawData)/4] IQR := Q3 - Q1 From 701483c896efb43fac384c151ddd30fe825e88d9 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 00:44:25 -0700 Subject: [PATCH 76/88] rm logstash benchmark case --- .../case.feature | 12 ------------ .../docker-compose.yaml | 12 ------------ .../logstash.conf | 15 --------------- 3 files changed, 39 deletions(-) delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml delete mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature deleted file mode 100644 index 96eef43009..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/case.feature +++ /dev/null @@ -1,12 +0,0 @@ -@input -Feature: performance file to blackhole logstash - Performance file to blackhole logstash - - @e2e-performance @docker-compose - Scenario: PerformanceFileToBlackholeLogstash - Given {docker-compose} environment - Given docker-compose boot type {benchmark} - When start docker-compose {performance_file_to_blackhole_logstash} - When start monitor {logstash} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} - When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml deleted file mode 100644 index 54f9d88678..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/docker-compose.yaml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.8' - -services: - - logstash: - image: docker.elastic.co/logstash/logstash:8.15.3 - volumes: - - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf - - .:/home/logstash - restart: always - - diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf b/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf deleted file mode 100644 index a006b5b2c3..0000000000 --- a/test/benchmark/test_cases/performance_file_to_blackhole_logstash/logstash.conf +++ /dev/null @@ -1,15 +0,0 @@ -input { - file { - path => "/home/logstash/*.log" - start_position => "beginning" - } -} -filter { - grok { - match => { "message" => "%{IP:ip} %{DATA:ident} %{DATA:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{DATA:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} (?:%{NUMBER:bytes}|-) \"%{DATA:referrer}\" \"%{DATA:user_agent}\"" } - } - drop {} -} -output { - stdout {} -} \ No newline at end of file From 5632bdd35b3cab9ca080326e14d22817d7208769 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 01:19:15 -0700 Subject: [PATCH 77/88] resolve github benchmark bug --- .github/workflows/benchmark.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 99756888fe..96162b4609 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -85,7 +85,9 @@ jobs: env: BUILD_LOGTAIL_UT: OFF WITHOUTGDB: ON - run: make benchmark + run: | + make benchmark + git stash - name: Store benchmark result iLogtail uses: benchmark-action/github-action-benchmark@v1 From 4f41d948423f8fce8cd810c1adc701695509d761 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 01:50:44 -0700 Subject: [PATCH 78/88] update monitor stop logic --- test/engine/setup/monitor/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 09bf9595f7..111c3cd85c 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -101,7 +101,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) copy(cpuRawData, m.statistic.GetCPURawData()) sort.Float64s(cpuRawData) - cpuRawData = cpuRawData[len(cpuRawData)/4 : 3*len(cpuRawData)/4+1] + cpuRawData = cpuRawData[len(cpuRawData)/4:] Q1 := cpuRawData[len(cpuRawData)/4] Q3 := cpuRawData[3*len(cpuRawData)/4] IQR := Q3 - Q1 From 59b556ca999186144b8bf8ff8376fb9e789120d3 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 05:33:58 -0700 Subject: [PATCH 79/88] update monitor.go --- test/engine/setup/monitor/monitor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 111c3cd85c..9d3264e4e3 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -101,7 +101,6 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) copy(cpuRawData, m.statistic.GetCPURawData()) sort.Float64s(cpuRawData) - cpuRawData = cpuRawData[len(cpuRawData)/4:] Q1 := cpuRawData[len(cpuRawData)/4] Q3 := cpuRawData[3*len(cpuRawData)/4] IQR := Q3 - Q1 From 5bac0d036a5b5fab836bd172a0a318a215cd4e7c Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 05:59:11 -0700 Subject: [PATCH 80/88] update monitor.go --- test/engine/setup/monitor/monitor.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 9d3264e4e3..a4e233d4ab 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -101,10 +101,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { cpuRawData := make([]float64, len(m.statistic.GetCPURawData())) copy(cpuRawData, m.statistic.GetCPURawData()) sort.Float64s(cpuRawData) - Q1 := cpuRawData[len(cpuRawData)/4] - Q3 := cpuRawData[3*len(cpuRawData)/4] - IQR := Q3 - Q1 - lowThreshold = Q1 - 1.5*IQR + lowThreshold = cpuRawData[len(cpuRawData)/2] * 0.3 fmt.Println("median of CPU usage rate(%):", cpuRawData[len(cpuRawData)/2]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) if lowThreshold < 0 { From 40824733d40869b264dae59bc67fadd120ca58e7 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 06:04:57 -0700 Subject: [PATCH 81/88] update monitor.go --- test/engine/setup/monitor/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index a4e233d4ab..d65d716d04 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -104,7 +104,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { lowThreshold = cpuRawData[len(cpuRawData)/2] * 0.3 fmt.Println("median of CPU usage rate(%):", cpuRawData[len(cpuRawData)/2]) fmt.Println("Low threshold of CPU usage rate(%):", lowThreshold) - if lowThreshold < 0 { + if lowThreshold < 1 { m.isMonitoring.Store(false) m.stopCh <- exitCodeErrorNotProcessing m.statistic.ClearStatistic() From d9b93818575a1dd5ba95c9d31a4a90442ac0d701 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 07:04:32 -0700 Subject: [PATCH 82/88] update github-action benchmark.yaml --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 96162b4609..4849d756da 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -101,7 +101,7 @@ jobs: - name: Store benchmark result other agents run: | git checkout gh-pages - cp test/benchmark/report/records_all.json dev/bench/records_all.js + mv test/benchmark/report/records_all.json dev/bench/records_all.js sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js git add dev/bench/records_all.js last_commit_message=$(git log -1 --pretty=%B) From 5103da0d3301d173c190a8f91d6b86437ca435c4 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 07:06:36 -0700 Subject: [PATCH 83/88] update monitor.go --- test/engine/setup/monitor/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index d65d716d04..a9743d75f9 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -123,7 +123,7 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { m.statistic.UpdateStatistic(stat) } cpuRawData := m.statistic.GetCPURawData() - if (cpuRawData[len(cpuRawData)-1] < lowThreshold) && (lowThreshold > 0) { + if cpuRawData[len(cpuRawData)-1] < lowThreshold { outlierCnt++ } if outlierCnt > 5 { From 229993f5f2955f8ff3ab68e67b31a445db23b395 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 07:47:04 -0700 Subject: [PATCH 84/88] update benchmark.yaml --- .github/workflows/benchmark.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 4849d756da..5cc0ff4411 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,15 +26,12 @@ on: - 'changes/**' - 'licenses/**' - 'CHANGELOG.md' - # types: - # - closed - push: - branches: - - benchmark + types: + - closed jobs: CI: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -108,12 +105,11 @@ jobs: git -c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com commit --amend -m "$last_commit_message" # TODO: Push gh-pages branch by github-action-bot - - name: Push benchmark result + # - name: Push benchmark result # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages - run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages result: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ubuntu-latest timeout-minutes: 60 needs: [ CI ] From b8269f45d0c7b726546b274aa1693073e9489680 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 17:46:39 -0700 Subject: [PATCH 85/88] update monitor.go --- test/engine/setup/monitor/monitor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index a9743d75f9..183df3ec14 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -125,6 +125,8 @@ func (m *Monitor) monitoring(client *client.Client, containerName string) { cpuRawData := m.statistic.GetCPURawData() if cpuRawData[len(cpuRawData)-1] < lowThreshold { outlierCnt++ + } else { + outlierCnt = 0 } if outlierCnt > 5 { bytes, _ := m.statistic.MarshalStatisticJSON() From 3793bf9e608abfcbe2714e9190f018797cec67b6 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 17:59:28 -0700 Subject: [PATCH 86/88] fix license --- licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md b/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md index a0cdaa4078..e0b2d8fa91 100644 --- a/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md +++ b/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md @@ -8,7 +8,6 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/pingcap/failpoint](https://pkg.go.dev/github.com/pingcap/failpoint?tab=licenses) - [github.com/pingcap/tidb/pkg/parser](https://pkg.go.dev/github.com/pingcap/tidb/pkg/parser?tab=licenses) - [sigs.k8s.io/gateway-api](https://pkg.go.dev/sigs.k8s.io/gateway-api?tab=licenses) -- [cloud.google.com/go/compute/metadata](https://pkg.go.dev/cloud.google.com/go/compute/metadata?tab=licenses) - [github.com/containerd/cgroups](https://pkg.go.dev/github.com/containerd/cgroups?tab=licenses) - [github.com/containerd/containerd](https://pkg.go.dev/github.com/containerd/containerd?tab=licenses) - [github.com/containerd/continuity](https://pkg.go.dev/github.com/containerd/continuity?tab=licenses) @@ -60,7 +59,6 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/prometheus/prometheus](https://pkg.go.dev/github.com/prometheus/prometheus?tab=licenses) - [github.com/stefanberger/go-pkcs11uri](https://pkg.go.dev/github.com/stefanberger/go-pkcs11uri?tab=licenses) - [github.com/tklauser/numcpus](https://pkg.go.dev/github.com/tklauser/numcpus?tab=licenses) -- [github.com/VictoriaMetrics/metricsql](https://pkg.go.dev/github.com/VictoriaMetrics/metricsql?tab=licenses) - [github.com/vishvananda/netlink](https://pkg.go.dev/github.com/vishvananda/netlink?tab=licenses) - [github.com/vishvananda/netns](https://pkg.go.dev/github.com/vishvananda/netns?tab=licenses) - [github.com/xdg-go/pbkdf2](https://pkg.go.dev/github.com/xdg-go/pbkdf2?tab=licenses) @@ -102,6 +100,7 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [go.opencensus.io](https://pkg.go.dev/go.opencensus.io?tab=licenses) - [github.com/golang/groupcache](https://pkg.go.dev/github.com/golang/groupcache?tab=licenses) - [sigs.k8s.io/controller-runtime](https://pkg.go.dev/sigs.k8s.io/controller-runtime?tab=licenses) +- [github.com/iLogtail/VictoriaMetrics fork from github.com/VictoriaMetrics/VictoriaMetrics](http://github.com/iLogtail/VictoriaMetrics) based on Apache-2.0 ## BSD licenses @@ -155,7 +154,6 @@ When distributed in a binary form, iLogtail may contain portions of the followin ## MIT licenses -- [github.com/valyala/gozstd](https://pkg.go.dev/github.com/valyala/gozstd?tab=licenses) - [github.com/goccy/go-json](https://pkg.go.dev/github.com/goccy/go-json?tab=licenses) - [github.com/gofrs/uuid](https://pkg.go.dev/github.com/gofrs/uuid?tab=licenses) - [github.com/go-kit/log](https://pkg.go.dev/github.com/go-kit/log?tab=licenses) @@ -166,7 +164,6 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/Microsoft/go-winio](https://pkg.go.dev/github.com/Microsoft/go-winio?tab=licenses) - [github.com/Microsoft/hcsshim](https://pkg.go.dev/github.com/Microsoft/hcsshim?tab=licenses) - [github.com/IBM/sarama](https://pkg.go.dev/github.com/IBM/sarama?tab=licenses) -- [github.com/VictoriaMetrics/fasthttp](https://pkg.go.dev/github.com/VictoriaMetrics/fasthttp?tab=licenses) - [github.com/buger/jsonparser](https://pkg.go.dev/github.com/buger/jsonparser?tab=licenses) - [github.com/cespare/xxhash](https://pkg.go.dev/github.com/cespare/xxhash?tab=licenses) - [github.com/eapache/go-resiliency](https://pkg.go.dev/github.com/eapache/go-resiliency?tab=licenses) @@ -201,11 +198,6 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/sirupsen/logrus](https://pkg.go.dev/github.com/sirupsen/logrus?tab=licenses) - [github.com/tchap/go-patricia](https://pkg.go.dev/github.com/tchap/go-patricia?tab=licenses) - [github.com/valyala/bytebufferpool](https://pkg.go.dev/github.com/valyala/bytebufferpool?tab=licenses) -- [github.com/valyala/fastjson](https://pkg.go.dev/github.com/valyala/fastjson?tab=licenses) -- [github.com/valyala/fastrand](https://pkg.go.dev/github.com/valyala/fastrand?tab=licenses) -- [github.com/valyala/fasttemplate](https://pkg.go.dev/github.com/valyala/fasttemplate?tab=licenses) -- [github.com/valyala/histogram](https://pkg.go.dev/github.com/valyala/histogram?tab=licenses) -- [github.com/valyala/quicktemplate](https://pkg.go.dev/github.com/valyala/quicktemplate?tab=licenses) - [github.com/99designs/keyring](https://pkg.go.dev/github.com/99designs/keyring?tab=licenses) - [github.com/danieljoos/wincred](https://pkg.go.dev/github.com/danieljoos/wincred?tab=licenses) - [github.com/dvsekhvalnov/jose2go](https://pkg.go.dev/github.com/dvsekhvalnov/jose2go?tab=licenses) @@ -258,9 +250,7 @@ When distributed in a binary form, iLogtail may contain portions of the followin ## iLogtail used or modified source code from these projects -- [github.com/iLogtail/VictoriaMetrics fork from github.com/VictoriaMetrics/VictoriaMetrics](http://github.com/iLogtail/VictoriaMetrics) based on Apache-2.0 - [github.com/iLogtail/jfr-parser fork from github.com/pyroscope-io/jfr-parser](http://github.com/iLogtail/jfr-parser) based on Apache-2.0 -- [github.com/iLogtail/metrics fork from github.com/VictoriaMetrics/metrics](http://github.com/iLogtail/metrics) based on MIT - [github.com/iLogtail/pyroscope-lib fork from github.com/pyroscope-io/pyroscope](http://github.com/iLogtail/pyroscope-lib) based on Apache-2.0 - [github.com/iLogtail/gonvml fork from github.com/mindprince/gonvml](https://github.com/iLogtail/gonvml) based on Apache-2.0 - [github.com/iLogtail/handy fork from github.com/streadway/handy](http://github.com/iLogtail/handy) based on BSD-2-Clause From 9403887d170aa10ac23d7668325aac67f1967686 Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 18:32:14 -0700 Subject: [PATCH 87/88] resolve conflict --- plugins/input/prometheus/input_prometheus.go | 190 +++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 plugins/input/prometheus/input_prometheus.go diff --git a/plugins/input/prometheus/input_prometheus.go b/plugins/input/prometheus/input_prometheus.go new file mode 100644 index 0000000000..0ecd38100c --- /dev/null +++ b/plugins/input/prometheus/input_prometheus.go @@ -0,0 +1,190 @@ +// Copyright 2021 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + liblogger "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" + + "github.com/alibaba/ilogtail/pkg/config" + "github.com/alibaba/ilogtail/pkg/logger" + "github.com/alibaba/ilogtail/pkg/pipeline" +) + +var libLoggerOnce sync.Once + +type ServiceStaticPrometheus struct { + Yaml string `comment:"the prometheus configuration content, more details please see [here](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)"` + ConfigFilePath string `comment:"the prometheus configuration path, and the param would be ignored when Yaml param is configured."` + AuthorizationPath string `comment:"the prometheus authorization path, only using in authorization files. When Yaml param is configured, the default value is the current binary path. However, the default value is the ConfigFilePath directory when ConfigFilePath is working."` + ExtraFlags map[string]string `comment:"the prometheus extra configuration flags, like promscrape.maxScrapeSize, for more flags please see [here](https://docs.victoriametrics.com/vmagent.html#advanced-usage)"` + NoStaleMarkers bool `comment:"Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series"` + + scraper *promscrape.Scraper //nolint:typecheck + shutdown chan struct{} + waitGroup sync.WaitGroup + context pipeline.Context + lock sync.Mutex + running bool + collector pipeline.Collector + kubeMeta *KubernetesMeta +} + +func (p *ServiceStaticPrometheus) Init(context pipeline.Context) (int, error) { + p.kubeMeta = NewKubernetesMeta(context) + if p.kubeMeta.readKubernetesWorkloadMeta() { + promscrape.ConfigMemberInfo(int(p.kubeMeta.replicas), strconv.Itoa(p.kubeMeta.currentNum)) + } + libLoggerOnce.Do(func() { + if f := flag.Lookup("loggerOutput"); f != nil { + _ = f.Value.Set("stdout") + } + // set max scrape size to 256MB + err := flag.Set("promscrape.maxScrapeSize", "268435456") + logger.Info(context.GetRuntimeContext(), "set config maxScrapeSize to 256MB, error", err) + liblogger.Init() + common.StartUnmarshalWorkers() + if p.NoStaleMarkers { + err := flag.Set("promscrape.noStaleMarkers", "true") + logger.Info(context.GetRuntimeContext(), "set config", "promscrape.noStaleMarkers", "value", "true", "error", err) + } + for k, v := range p.ExtraFlags { + err := flag.Set(k, v) + logger.Info(context.GetRuntimeContext(), "set config", k, "value", v, "error", err) + } + }) + p.context = context + var detail []byte + switch { + case p.Yaml != "": + detail = []byte(p.Yaml) + if p.AuthorizationPath == "" { + p.AuthorizationPath = config.LoongcollectorGlobalConfig.LoongcollectorConfDir + } + case p.ConfigFilePath != "": + f, err := os.Open(p.ConfigFilePath) + if err != nil { + return 0, fmt.Errorf("cannot find prometheus configuration file") + } + defer func(f *os.File) { + _ = f.Close() + }(f) + bytes, err := io.ReadAll(f) + if err != nil { + return 0, fmt.Errorf("cannot read prometheus configuration file") + } + detail = bytes + if p.AuthorizationPath == "" { + p.AuthorizationPath = filepath.Dir(p.ConfigFilePath) + } + default: + return 0, errors.New("the scrape configuration is required") + } + var err error + if p.AuthorizationPath, err = filepath.Abs(p.AuthorizationPath); err != nil { + return 0, fmt.Errorf("cannot find the abs authorization path: %v", err) + } + name := strings.Join([]string{context.GetProject(), context.GetLogstore(), context.GetConfigName()}, "_") + p.scraper = promscrape.NewScraper(detail, name, p.AuthorizationPath) //nolint:typecheck + if err := p.scraper.CheckConfig(); err != nil { + return 0, fmt.Errorf("illegal prometheus configuration file %s: %v", name, err) + } + return 0, nil +} + +func (p *ServiceStaticPrometheus) Description() string { + return "prometheus scrape plugin for logtail, use vmagent lib" +} + +// Start starts the ServiceInput's service, whatever that may be +func (p *ServiceStaticPrometheus) Start(c pipeline.Collector) error { + p.collector = c + p.shutdown = make(chan struct{}) + p.waitGroup.Add(1) + defer p.waitGroup.Done() + p.scraper.Init(p.slsPushData) + p.running = true + if p.kubeMeta.isWorkingOnClusterMode() { + p.StartKubeReloadScraper() + } + <-p.shutdown + p.scraper.Stop() + return nil +} + +// Stop stops the services and closes any necessary channels and connections +func (p *ServiceStaticPrometheus) Stop() error { + p.lock.Lock() + defer p.lock.Unlock() + p.running = false + close(p.shutdown) + p.waitGroup.Wait() + return nil +} + +func (p *ServiceStaticPrometheus) StartKubeReloadScraper() { + go func() { + ticker := time.NewTicker(time.Second * 10) + for { + select { + case <-p.shutdown: + return + case <-ticker.C: + change, err := p.kubeMeta.getPrometheusReplicas() + if !change || err != nil { + continue + } + logger.Info(p.context.GetRuntimeContext(), "found change replicas, would start reload prometheus scraper", p.kubeMeta.replicas) + p.lock.Lock() + if !p.running { + return + } + promscrape.ConfigMemberInfo(int(p.kubeMeta.replicas), strconv.Itoa(p.kubeMeta.currentNum)) + p.scraper.Stop() + p.scraper.Init(p.slsPushData) + p.lock.Unlock() + logger.Info(p.context.GetRuntimeContext(), "reload prometheus scraper done") + } + } + }() +} + +func init() { + pipeline.ServiceInputs["service_prometheus"] = func() pipeline.ServiceInput { + return &ServiceStaticPrometheus{ + NoStaleMarkers: true, + } + } +} + +func (p *ServiceStaticPrometheus) slsPushData(_ *auth.Token, wr *prompbmarshal.WriteRequest) { + logger.Debug(p.context.GetRuntimeContext(), "append new metrics", wr.Size()) + appendTSDataToSlsLog(p.collector, wr) + logger.Debug(p.context.GetRuntimeContext(), "append done", wr.Size()) +} From 413b6f7e60a64b5d842b80e4696aad19f885a8bd Mon Sep 17 00:00:00 2001 From: QianyongXu <13038089398@163.com> Date: Thu, 24 Oct 2024 20:55:58 -0700 Subject: [PATCH 88/88] resolve lint --- licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md | 10 ++++++++++ .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../performance_file_to_blackhole_vector/case.feature | 2 +- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md b/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md index e0b2d8fa91..f8814a01d8 100644 --- a/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md +++ b/licenses/LICENSE_OF_ILOGTAIL_DEPENDENCIES.md @@ -101,6 +101,8 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/golang/groupcache](https://pkg.go.dev/github.com/golang/groupcache?tab=licenses) - [sigs.k8s.io/controller-runtime](https://pkg.go.dev/sigs.k8s.io/controller-runtime?tab=licenses) - [github.com/iLogtail/VictoriaMetrics fork from github.com/VictoriaMetrics/VictoriaMetrics](http://github.com/iLogtail/VictoriaMetrics) based on Apache-2.0 +- [cloud.google.com/go/compute/metadata](https://pkg.go.dev/cloud.google.com/go/compute/metadata?tab=licenses) +- [github.com/VictoriaMetrics/metricsql](https://pkg.go.dev/github.com/VictoriaMetrics/metricsql?tab=licenses) ## BSD licenses @@ -229,6 +231,13 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/stretchr/testify](https://pkg.go.dev/github.com/stretchr/testify?tab=licenses) - [go.uber.org/goleak](https://pkg.go.dev/go.uber.org/goleak?tab=licenses) - [github.com/influxdata/telegraf](https://pkg.go.dev/github.com/influxdata/telegraf?tab=licenses) +- [github.com/valyala/fastjson](https://pkg.go.dev/github.com/valyala/fastjson?tab=licenses) +- [github.com/valyala/fastrand](https://pkg.go.dev/github.com/valyala/fastrand?tab=licenses) +- [github.com/valyala/fasttemplate](https://pkg.go.dev/github.com/valyala/fasttemplate?tab=licenses) +- [github.com/valyala/gozstd](https://pkg.go.dev/github.com/valyala/gozstd?tab=licenses) +- [github.com/valyala/histogram](https://pkg.go.dev/github.com/valyala/histogram?tab=licenses) +- [github.com/valyala/quicktemplate](https://pkg.go.dev/github.com/valyala/quicktemplate?tab=licenses) +- [github.com/VictoriaMetrics/fasthttp](https://pkg.go.dev/github.com/VictoriaMetrics/fasthttp?tab=licenses) ## ISC licenses @@ -254,3 +263,4 @@ When distributed in a binary form, iLogtail may contain portions of the followin - [github.com/iLogtail/pyroscope-lib fork from github.com/pyroscope-io/pyroscope](http://github.com/iLogtail/pyroscope-lib) based on Apache-2.0 - [github.com/iLogtail/gonvml fork from github.com/mindprince/gonvml](https://github.com/iLogtail/gonvml) based on Apache-2.0 - [github.com/iLogtail/handy fork from github.com/streadway/handy](http://github.com/iLogtail/handy) based on BSD-2-Clause +- [github.com/iLogtail/metrics fork from github.com/VictoriaMetrics/metrics](http://github.com/iLogtail/metrics) based on MIT diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index ad35f0ec00..b8bbf86d1a 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 96d7fa9b36..449511f10d 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 08ab9a205a..92bb93f5d6 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature index 03731669a4..327b8d27a1 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtailspl} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index bc3fd8d10f..b0e54b85c1 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate random nginx logs to file, speed {10}MB/s, total {1}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} When wait monitor until log processing finished