From b218e0204c5e5d8aa7ae8818bc3b50000beb234a Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 6 Nov 2024 13:59:18 +0800 Subject: [PATCH 1/9] test: refactor E2E trigger --- .github/workflows/benchmark.yaml | 7 + docker/Dockerfile.e2e-test | 5 +- test/config/config.go | 3 + test/engine/setup/docker_compose.go | 28 ++- test/engine/steps.go | 36 ++-- .../trigger/{protocol.go => ebpf/http.go} | 4 +- .../{ebpf_trigger.go => ebpf/security.go} | 27 +-- test/engine/trigger/file.go | 114 ---------- test/engine/trigger/generator/apsara_test.go | 73 ------- .../trigger/generator/delimiter_test.go | 127 ----------- .../trigger/generator/ebpf_file_mmap_test.go | 51 ----- test/engine/trigger/generator/helper.go | 85 -------- test/engine/trigger/generator/json_test.go | 119 ----------- test/engine/trigger/generator/regex_test.go | 198 ------------------ test/engine/trigger/helper.go | 21 +- test/engine/trigger/log/file.go | 79 +++++++ test/engine/trigger/{ => log}/http.go | 2 +- test/engine/trigger/remote/log.py | 140 +++++++++++++ test/engine/trigger/remote/log_benchmark.py | 58 +++++ test/engine/trigger/remote/mmap.py | 22 ++ test/engine/trigger/trigger.go | 96 --------- test/requirements.txt | 1 + 22 files changed, 391 insertions(+), 905 deletions(-) rename test/engine/trigger/{protocol.go => ebpf/http.go} (90%) rename test/engine/trigger/{ebpf_trigger.go => ebpf/security.go} (76%) delete mode 100644 test/engine/trigger/file.go delete mode 100644 test/engine/trigger/generator/apsara_test.go delete mode 100644 test/engine/trigger/generator/delimiter_test.go delete mode 100644 test/engine/trigger/generator/ebpf_file_mmap_test.go delete mode 100644 test/engine/trigger/generator/helper.go delete mode 100644 test/engine/trigger/generator/json_test.go delete mode 100644 test/engine/trigger/generator/regex_test.go create mode 100644 test/engine/trigger/log/file.go rename test/engine/trigger/{ => log}/http.go (99%) create mode 100644 test/engine/trigger/remote/log.py create mode 100644 test/engine/trigger/remote/log_benchmark.py create mode 100644 test/engine/trigger/remote/mmap.py delete mode 100644 test/engine/trigger/trigger.go create mode 100644 test/requirements.txt diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 5cc0ff4411..2f29670fca 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -37,6 +37,7 @@ jobs: strategy: matrix: go-version: [ 1.19.10 ] + python-version: [ 3.8 ] runner: [ ubuntu-latest ] fail-fast: true permissions: @@ -62,6 +63,11 @@ jobs: with: go-version: ${{ matrix.go-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Check out code uses: actions/checkout@v2 with: @@ -83,6 +89,7 @@ jobs: BUILD_LOGTAIL_UT: OFF WITHOUTGDB: ON run: | + pip3 install -r test/requirements.txt make benchmark git stash diff --git a/docker/Dockerfile.e2e-test b/docker/Dockerfile.e2e-test index 785df4caa3..e2acd47d42 100644 --- a/docker/Dockerfile.e2e-test +++ b/docker/Dockerfile.e2e-test @@ -1,12 +1,11 @@ -FROM golang:1.19 +FROM python:3.8 -RUN go env -w GOPROXY="https://goproxy.cn,direct" RUN mkdir -p /tmp/loongcollector WORKDIR /root COPY . ./loongcollector WORKDIR /root/loongcollector/test -RUN go mod download +RUN pip3 install -r requirements.txt CMD ["sh", "-c", "while true; do sleep 3600; done"] \ No newline at end of file diff --git a/test/config/config.go b/test/config/config.go index 915bd08c6d..76c9df0e73 100644 --- a/test/config/config.go +++ b/test/config/config.go @@ -72,6 +72,9 @@ func ParseConfig() { TestConfig.GeneratedLogDir = "/tmp/loongcollector" } TestConfig.WorkDir = os.Getenv("WORK_DIR") + if len(TestConfig.WorkDir) == 0 { + TestConfig.WorkDir, _ = os.Getwd() + } // SSH TestConfig.SSHUsername = os.Getenv("SSH_USERNAME") diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 6a7e3fc4de..49dc7538ca 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "time" @@ -128,5 +129,30 @@ func (d *DockerComposeEnv) ExecOnLogtail(command string) (string, error) { } func (d *DockerComposeEnv) ExecOnSource(ctx context.Context, command string) (string, error) { - return "", fmt.Errorf("not implemented") + // exec on host of docker compose + cmd := exec.Command(command) + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return "", err + } + if err := cmd.Start(); err != nil { + return "", err + } + buf := make([]byte, 1024) + n, _ := stdout.Read(buf) + if n > 0 { + return string(buf[:n]), nil + } + n, _ = stderr.Read(buf) + if n > 0 { + return string(buf[:n]), nil + } + if err := cmd.Wait(); err != nil { + return "", err + } + return "", nil } diff --git a/test/engine/steps.go b/test/engine/steps.go index 7b19db1988..336a5ff8ac 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -10,8 +10,11 @@ import ( "github.com/alibaba/ilogtail/test/engine/cleanup" "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" "github.com/alibaba/ilogtail/test/engine/trigger" + "github.com/alibaba/ilogtail/test/engine/trigger/ebpf" + "github.com/alibaba/ilogtail/test/engine/trigger/log" "github.com/alibaba/ilogtail/test/engine/verify" ) @@ -27,6 +30,7 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig) ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) ctx.Given(`^mkdir \{(.*)\}`, setup.Mkdir) + ctx.Given(`^docker-compose boot type \{(\S+)\}$`, setup.SetDockerComposeBootType) // ------------------------------------------ // When @@ -41,19 +45,25 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { // generate ctx.When(`^begin trigger`, trigger.BeginTrigger) - ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingle) - ctx.When(`^generate \{(\d+)\} multiline regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexMultiline) - ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingleGBK) - ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) - ctx.When(`^generate \{(\d+)\} apsara logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.Apsara) - ctx.When(`^generate \{(\d+)\} delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterSingle) - ctx.When(`^generate \{(\d+)\} multiline delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterMultiline) - ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONSingle) - ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONMultiline) - ctx.When(`^execute \{(\d+)\} commands to generate process security events`, trigger.TrigerProcessSecurityEvents) - ctx.When(`^execute \{(\d+)\} commands to generate network security events on url \{(.*)\}$`, trigger.TrigerNetworksSecurityEvents) - ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) - ctx.When(`^generate \{(\d+)\} HTTP requests, with interval \{(\d+)\}ms, url: \{(.*)\}`, trigger.TrigerHTTP) + // log + ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexSingle) + ctx.When(`^generate \{(\d+)\} multiline regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexMultiline) + ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexSingleGBK) + ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, log.HTTP) + ctx.When(`^generate \{(\d+)\} apsara logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.Apsara) + ctx.When(`^generate \{(\d+)\} delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, log.DelimiterSingle) + ctx.When(`^generate \{(\d+)\} multiline delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, log.DelimiterMultiline) + ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.JSONSingle) + ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.JSONMultiline) + ctx.When(`^generate random nginx logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, log.Nginx) + ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) + ctx.When(`^wait monitor until log processing finished$`, monitor.WaitMonitorUntilProcessingFinished) + + // ebpf + ctx.When(`^execute \{(\d+)\} commands to generate process security events`, ebpf.ProcessSecurityEvents) + ctx.When(`^execute \{(\d+)\} commands to generate network security events on url \{(.*)\}$`, ebpf.NetworksSecurityEvents) + ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, ebpf.FileSecurityEvents) + ctx.When(`^generate \{(\d+)\} HTTP requests, with interval \{(\d+)\}ms, url: \{(.*)\}`, ebpf.HTTP) // ------------------------------------------ // Then diff --git a/test/engine/trigger/protocol.go b/test/engine/trigger/ebpf/http.go similarity index 90% rename from test/engine/trigger/protocol.go rename to test/engine/trigger/ebpf/http.go index 646a4aa97b..c3c84f9ddd 100644 --- a/test/engine/trigger/protocol.go +++ b/test/engine/trigger/ebpf/http.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package ebpf import ( "context" @@ -22,7 +22,7 @@ import ( "github.com/alibaba/ilogtail/test/engine/setup" ) -func TrigerHTTP(ctx context.Context, count int, interval int, url string) (context.Context, error) { +func HTTP(ctx context.Context, count int, interval int, url string) (context.Context, error) { logger.Debugf(context.Background(), "count:%d interval:%d url:%s", count, interval, url) cmd := fmt.Sprintf("curl -vL %s", url) time.Sleep(time.Second * 5) diff --git a/test/engine/trigger/ebpf_trigger.go b/test/engine/trigger/ebpf/security.go similarity index 76% rename from test/engine/trigger/ebpf_trigger.go rename to test/engine/trigger/ebpf/security.go index 4b824c557e..9cc85d527b 100644 --- a/test/engine/trigger/ebpf_trigger.go +++ b/test/engine/trigger/ebpf/security.go @@ -11,16 +11,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package ebpf import ( "context" - "html/template" + "strconv" "strings" "time" - "github.com/alibaba/ilogtail/test/config" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/trigger" ) /* @@ -28,7 +28,7 @@ import ( input_process_security ******************** */ -func TrigerProcessSecurityEvents(ctx context.Context, commandCnt int) (context.Context, error) { +func ProcessSecurityEvents(ctx context.Context, commandCnt int) (context.Context, error) { time.Sleep(5 * time.Second) if err := execveCommands(ctx, commandCnt); err != nil { return ctx, err @@ -51,7 +51,7 @@ func execveCommands(ctx context.Context, commandCnt int) error { input_network_security ******************** */ -func TrigerNetworksSecurityEvents(ctx context.Context, commandCnt int, url string) (context.Context, error) { +func NetworksSecurityEvents(ctx context.Context, commandCnt int, url string) (context.Context, error) { time.Sleep(5 * time.Second) if err := curlURL(ctx, commandCnt, url); err != nil { return ctx, err @@ -74,9 +74,8 @@ func curlURL(ctx context.Context, commandCnt int, url string) error { input_file_security ******************** */ -const triggerFileSecurityTemplate = "cd {{.WorkDir}} && COMMAND_CNT={{.CommandCnt}} FILE_NAME={{.FileName}} {{.Command}}" -func TrigerFileSecurityEvents(ctx context.Context, commandCnt int, filenames string) (context.Context, error) { +func FileSecurityEvents(ctx context.Context, commandCnt int, filenames string) (context.Context, error) { time.Sleep(5 * time.Second) if err := rwFile(ctx, commandCnt, filenames); err != nil { return ctx, err @@ -112,20 +111,10 @@ func rwFile(ctx context.Context, commandCnt int, filenames string) error { } func mmapFile(ctx context.Context, commandCnt int, filenames string) error { - mmapFileCommand := getRunTriggerCommand("TestGenerateMmapCommand") files := strings.Split(filenames, ",") for _, file := range files { - var triggerEBPFCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerFileSecurityTemplate)) - if err := template.Execute(&triggerEBPFCommand, map[string]interface{}{ - "WorkDir": config.TestConfig.WorkDir, - "CommandCnt": commandCnt, - "FileName": file, - "Command": mmapFileCommand, - }); err != nil { - return err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerEBPFCommand.String()); err != nil { + mmapFileCommand := trigger.GetRunTriggerCommand("mmap", "commandCnt", strconv.FormatInt(int64(commandCnt), 10), "filename", file) + if _, err := setup.Env.ExecOnSource(ctx, mmapFileCommand); err != nil { return err } } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go deleted file mode 100644 index 373aebbe52..0000000000 --- a/test/engine/trigger/file.go +++ /dev/null @@ -1,114 +0,0 @@ -package trigger - -import ( - "context" - "fmt" - "math/rand" - "os" - "path/filepath" - "time" - - "golang.org/x/time/rate" - - "github.com/alibaba/ilogtail/test/config" -) - -// JSON template -func GenerateRandomNginxLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - - // clear file - path = filepath.Clean(path) - path = filepath.Join(config.CaseHome, path) - fmt.Println(path) - _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - - rand.Seed(time.Now().UnixNano()) - maxLogLen := 1024 - nginxLog := genNginxLog() - - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLogLen) - - timeout := time.After(time.Minute * time.Duration(totalTime)) - - for { - select { - // context is done - case <-ctx.Done(): - // clear file - _ = file.Close() - return ctx, nil - // all time is done - case <-timeout: - // clear file - _ = file.Close() - return ctx, nil - default: - if limiter.AllowN(time.Now(), len(nginxLog)) { - _, _ = file.WriteString(nginxLog + "\n") // #nosec G307 - nginxLog = genNginxLog() - } - } - } -} - -var ipAddresses = []string{ - "103.159.151.180", - "12.55.18.241", - "182.233.128.102", - "221.85.57.231", - "76.245.65.224", - "86.250.231.93", - "44.201.253.252", - "218.7.2.219", - "172.118.174.109", - "208.16.46.154", - "7.138.80.41", - "214.73.25.80", - "83.124.20.79", - "80.226.48.153", - "92.129.204.161", - "212.103.145.159", - "148.188.8.90", - "148.212.244.121", - "106.186.172.157", - "30.127.196.158", -} - -var userAgents = []string{ - "aliyun-sdk-java", - "aliyun-sdk-golang", - "aliyun-sdk-python", -} - -var statusCodes = []string{ - "400", - "401", - "402", - "403", - "404", - "200", -} - -const bytesMean = 5500.0 -const bytesStddev = 1500.0 - -func genNginxLog() string { - nginxLogTemplate := `%s - - [%s] "GET http://www.districtdot-com.biz/syndicate HTTP/1.1" %s %d "http://www.chiefscalable.biz/webservices" "%s"` - currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") - ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // #nosec G404 - statusIdx := rand.Intn(len(statusCodes) * 10) // #nosec G404 - if statusIdx >= len(statusCodes) { - statusIdx = len(statusCodes) - 1 - } - bytesSize := int32(rand.NormFloat64()*bytesStddev + bytesMean) - if bytesSize < 1000 { - bytesSize = 0 - } else if bytesSize > 10000 { - bytesSize = 10000 - } - statusCode := statusCodes[statusIdx] - userAgent := userAgents[rand.Intn(len(userAgents))] // #nosec G404 - - return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, bytesSize, userAgent) -} diff --git a/test/engine/trigger/generator/apsara_test.go b/test/engine/trigger/generator/apsara_test.go deleted file mode 100644 index 40329c0cfe..0000000000 --- a/test/engine/trigger/generator/apsara_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "strconv" - "testing" - "time" -) - -// TestGenerateApsara will be executed in the environment being collected. -func TestGenerateApsara(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/application/Application:12]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:hello world!\n", - "[{{.Time}}]\t[{{.Level}}]\t[20964]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:这是一条消息\n", - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:hello world!\n", - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:这是一条消息\n", - "[{{.Time}}]\t[{{.Level}}]\t[00001]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:password:123456\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - var currentTime string - if i%2 == 0 { - currentTime = time.Now().Format("2006-01-02 15:04:05.000000") - } else { - currentTime = strconv.FormatInt(time.Now().UnixNano()/1000, 10) - } - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": currentTime, - "Level": getRandomLogLevel(), - "LogNo": logNo + i, - "FileNo": fileNo, - "Mark": getRandomMark(), - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/delimiter_test.go b/test/engine/trigger/generator/delimiter_test.go deleted file mode 100644 index 354b6dea8c..0000000000 --- a/test/engine/trigger/generator/delimiter_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "testing" - "time" -) - -// TestGenerateDelimiterSingle will be executed in the environment being collected. -func TestGenerateDelimiterSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv("Delimiter", "Quote") - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - delimiter := config.Custom["Delimiter"] - if delimiter == "" { - delimiter = " " - } - quote := config.Custom["Quote"] - if quote == "" { - quote = "" - } - testLogContentTmpl := string2Template([]string{ - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}0.0.0.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/index.html{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}628{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}218{{.Quote}}{{.Delimiter}}{{.Quote}}go-sdk{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}744{{.Quote}}{{.Delimiter}}{{.Quote}}Mozilla/5.0{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}127.0.0.1{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}200{{.Quote}}{{.Delimiter}}{{.Quote}}320{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}192.168.0.3{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}949{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo, - "Time": time.Now().Format("2006-01-02 15:04:05.000000000"), - "Delimiter": delimiter, - "Quote": quote, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -// TestGenerateDelimiterMultiline will be executed in the environment being collected. -func TestGenerateDelimiterMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv("Delimiter", "Quote") - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - delimiter := config.Custom["Delimiter"] - if delimiter == "" { - delimiter = " " - } - quote := config.Custom["Quote"] - if quote == "" { - quote = "" - } - testLogContentTmpl := string2Template([]string{ - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}0.0.0.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/index.html{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}628{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}-{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}218{{.Quote}}{{.Delimiter}}{{.Quote}}go-sdk{{.Quote}}\n", - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}744{{.Quote}}{{.Delimiter}}{{.Quote}}Mozilla/5.0{{.Quote}}\n", - "{{.Quote}}-{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}127.0.0.1{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}200{{.Quote}}{{.Delimiter}}{{.Quote}}320{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}192.168.0.3{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}949{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "FileNo": fileNo, - "LogNo": logNo, - "Time": time.Now().Format("2006-01-02 15:04:05.000000000"), - "Delimiter": delimiter, - "Quote": quote, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/ebpf_file_mmap_test.go b/test/engine/trigger/generator/ebpf_file_mmap_test.go deleted file mode 100644 index 56e221b54d..0000000000 --- a/test/engine/trigger/generator/ebpf_file_mmap_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "os" - "strconv" - "syscall" - "testing" -) - -func TestGenerateMmapCommand(t *testing.T) { - commandCnt := getEnvOrDefault("COMMAND_CNT", "10") - commandCntNum, err := strconv.Atoi(commandCnt) - if err != nil { - t.Fatalf("parse COMMAND_CNT failed: %v", err) - return - } - filename := getEnvOrDefault("FILE_NAME", "/tmp/loongcollector/ebpfFileSecurityHook3.log") - f, err := os.Create(filename) - if err != nil { - panic(err) - } - fd := int(f.Fd()) - for i := 0; i < commandCntNum; i++ { - b, innerErr := syscall.Mmap(fd, 0, 20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) - if innerErr != nil { - panic(innerErr) - } - innerErr = syscall.Munmap(b) - if innerErr != nil { - panic(innerErr) - } - } - err = os.Remove(filename) - if err != nil { - t.Fatalf("remove file failed: %v", err) - return - } -} diff --git a/test/engine/trigger/generator/helper.go b/test/engine/trigger/generator/helper.go deleted file mode 100644 index 2514bb2614..0000000000 --- a/test/engine/trigger/generator/helper.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "crypto/rand" - "fmt" - "math/big" - "os" - "strconv" - "text/template" - - "github.com/pkg/errors" -) - -var Levels = []string{"ERROR", "INFO", "DEBUG", "WARNING"} - -type GenerateFileLogConfig struct { - GeneratedLogDir string - TotalLog int - Interval int - FileName string - Custom map[string]string -} - -func getGenerateFileLogConfigFromEnv(customKeys ...string) (*GenerateFileLogConfig, error) { - gneratedLogDir := getEnvOrDefault("GENERATED_LOG_DIR", "/tmp/loongcollector") - totalLog, err := strconv.Atoi(getEnvOrDefault("TOTAL_LOG", "100")) - if err != nil { - return nil, errors.Wrap(err, "parse TOTAL_LOG failed") - } - interval, err := strconv.Atoi(getEnvOrDefault("INTERVAL", "1")) - if err != nil { - return nil, errors.Wrap(err, "parse INTERVAL failed") - } - fileName := getEnvOrDefault("FILENAME", "default.log") - custom := make(map[string]string) - for _, key := range customKeys { - custom[key] = getEnvOrDefault(key, "") - } - return &GenerateFileLogConfig{ - GeneratedLogDir: gneratedLogDir, - TotalLog: totalLog, - Interval: interval, - FileName: fileName, - Custom: custom, - }, nil -} - -func string2Template(strings []string) []*template.Template { - templates := make([]*template.Template, len(strings)) - for i, str := range strings { - templates[i], _ = template.New(fmt.Sprintf("template_%d", i)).Parse(str) - } - return templates -} - -func getRandomLogLevel() string { - randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(Levels)))) - return Levels[randInt.Int64()] -} - -func getRandomMark() string { - marks := []string{"-", "F"} - randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(marks)))) - return marks[randInt.Int64()] -} - -func getEnvOrDefault(env, fallback string) string { - if value, ok := os.LookupEnv(env); ok { - return value - } - return fallback -} diff --git a/test/engine/trigger/generator/json_test.go b/test/engine/trigger/generator/json_test.go deleted file mode 100644 index 0b4ef51b33..0000000000 --- a/test/engine/trigger/generator/json_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "strconv" - "testing" - "time" -) - -// TestGenerateJSONSingle will be executed in the environment being collected. -func TestGenerateJSONSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"POST","userAgent":"mozilla firefox","size":263} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"GET","userAgent":"go-sdk","size":569} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"HEAD","userAgent":"go-sdk","size":210} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"192.168.0.3","method":"PUT","userAgent":"curl/7.10","size":267} -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - var currentTime string - if i%2 == 0 { - currentTime = time.Now().Format("2006-01-02T15:04:05.999999999") - } else { - currentTime = strconv.FormatInt(time.Now().UnixNano()/1000, 10) - } - testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Time": currentTime, - }) - - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -func TestGenerateJSONMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"POST","userAgent":"mozilla firefox", -"size":263} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"GET","userAgent":"go-sdk", -"size":569} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"HEAD","userAgent":"go-sdk", -"size":210} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"192.168.0.3","method":"PUT","userAgent":"curl/7.10", -"size":267} -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - currentTime := time.Now().Format("2006-01-02T15:04:05.999999999") - testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Time": currentTime, - }) - - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/regex_test.go b/test/engine/trigger/generator/regex_test.go deleted file mode 100644 index c50e65dc6e..0000000000 --- a/test/engine/trigger/generator/regex_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "os" - "testing" - "time" - - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" -) - -// TestGenerateRegexLogSingle will be executed in the environment being collected. -func TestGenerateRegexLogSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "HEAD / HTTP/2.0" 302 809 "未知" "这是一条消息，password:123456" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "GET /index.html HTTP/2.0" 200 139 "Mozilla/5.0" "这是一条消息，password:123456，这是第二条消息，password:00000" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 10.45.26.0 - [{{.Time}}] "PUT /index.html HTTP/1.1" 200 913 "curl/7.10" "这是一条消息" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 192.168.0.3 - [{{.Time}}] "PUT /dir/resource.txt HTTP/2.0" 501 355 "go-sdk" "这是一条消息，password:123456" -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -// TestGenerateRegexLogSingleGBK will be executed in the environment being collected. -func TestGenerateRegexLogSingleGBK(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - encoder := simplifiedchinese.GBK.NewEncoder() - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "HEAD / HTTP/2.0" 302 809 "未知" "这是一条消息，password:123456" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "GET /index.html HTTP/2.0" 200 139 "Mozilla/5.0" "这是一条消息，password:123456，这是第二条消息，password:00000" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 10.45.26.0 - [{{.Time}}] "PUT /index.html HTTP/1.1" 200 913 "curl/7.10" "这是一条消息" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 192.168.0.3 - [{{.Time}}] "PUT /dir/resource.txt HTTP/2.0" 501 355 "go-sdk" "这是一条消息，password:123456" -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - var buffer bytes.Buffer - _ = testLogContentTmpl[logIndex].Execute(&buffer, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - }) - data, err1 := io.ReadAll(transform.NewReader(&buffer, encoder)) - if err1 != nil { - t.Fatalf("encode log failed: %v", err1) - } - _, err := io.WriteString(file, string(data)) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -func TestGenerateRegexLogMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: exception happened -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:73) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: 发生异常 -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f2(RegexMultiLog.java:80) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:75) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: exception happened -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f5(RegexMultiLog.java:100) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f4(RegexMultiLog.java:96) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f3(RegexMultiLog.java:89) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f2(RegexMultiLog.java:82) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:75) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Level": getRandomLogLevel(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Mark": getRandomMark(), - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/helper.go b/test/engine/trigger/helper.go index 8e0fb12171..c8af4a3736 100644 --- a/test/engine/trigger/helper.go +++ b/test/engine/trigger/helper.go @@ -14,11 +14,26 @@ package trigger import ( + "context" "fmt" + "path/filepath" + "strings" + "time" + + "github.com/alibaba/ilogtail/test/config" ) -const commandTemplate = "/usr/local/go/bin/go test -count=1 -v -run ^%s$ github.com/alibaba/ilogtail/test/engine/trigger/generator" +const commandTemplate = "cd %s && python3 %s.py %s" + +func BeginTrigger(ctx context.Context) (context.Context, error) { + startTime := time.Now().Unix() + return context.WithValue(ctx, config.StartTimeContextKey, int32(startTime)), nil +} -func getRunTriggerCommand(triggerName string) string { - return fmt.Sprintf(commandTemplate, triggerName) +func GetRunTriggerCommand(triggerName string, kvs ...string) string { + args := make([]string, 0) + for i := 0; i < len(kvs); i += 2 { + args = append(args, fmt.Sprintf("--%s", kvs[i]), kvs[i+1]) + } + return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "test", "engine", "trigger", "remote"), triggerName, strings.Join(args, " ")) } diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go new file mode 100644 index 0000000000..dd1275e4bd --- /dev/null +++ b/test/engine/trigger/log/file.go @@ -0,0 +1,79 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package log + +import ( + "context" + "strconv" + "time" + + "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/trigger" +) + +func RegexSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regex", path, totalLog, interval) +} + +func RegexSingleGBK(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regexGBK", path, totalLog, interval) +} + +func RegexMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regexMultiline", path, totalLog, interval) +} + +func JSONSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "json", path, totalLog, interval) +} + +func JSONMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "jsonMultiline", path, totalLog, interval) +} + +func Apsara(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "apsara", path, totalLog, interval) +} + +func DelimiterSingle(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { + return generate(ctx, "delimiter", path, totalLog, interval, "delimiter", delimiter, "quote", quote) +} + +func DelimiterMultiline(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { + return generate(ctx, "delimiterMultiline", path, totalLog, interval, "delimiter", delimiter, "quote", quote) +} + +func Nginx(ctx context.Context, rate, duration int, path string) (context.Context, error) { + return generateBenchmark(ctx, "nginx", path, rate, duration) +} + +func generate(ctx context.Context, mode, path string, count, interval int, customKV ...string) (context.Context, error) { + time.Sleep(3 * time.Second) + customKV = append(customKV, "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval)) + command := trigger.GetRunTriggerCommand("log", customKV...) + if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { + return ctx, err + } + return ctx, nil +} + +func generateBenchmark(ctx context.Context, mode, path string, rate, duration int, customKV ...string) (context.Context, error) { + time.Sleep(3 * time.Second) + customKV = append(customKV, "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration)) + command := trigger.GetRunTriggerCommand("log_benchmark", customKV...) + if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { + return ctx, err + } + return ctx, nil +} diff --git a/test/engine/trigger/http.go b/test/engine/trigger/log/http.go similarity index 99% rename from test/engine/trigger/http.go rename to test/engine/trigger/log/http.go index 4a580e4c16..d2eec5b961 100644 --- a/test/engine/trigger/http.go +++ b/test/engine/trigger/log/http.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package log import ( "context" diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/remote/log.py new file mode 100644 index 0000000000..0490240983 --- /dev/null +++ b/test/engine/trigger/remote/log.py @@ -0,0 +1,140 @@ +import argparse +import logging +import random +import time + +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime +from faker import Faker +from faker.providers import internet, user_agent, lorem, misc + + +def apsara(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'[{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def delimiter(args, logger, faker): + custom_args = args.custom + quote = custom_args.get('quote', '') + delimiter = custom_args.get('delimiter', ' ') + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def delimiterMultiline(args, logger, faker): + custom_args = args.custom + quote = custom_args.get('quote', '') + delimiter = custom_args.get('delimiter', ' ') + fileNo = random.randint(1, 1000) + for i in range(args.count): + log = f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}' + breakLineIdx1 = random.randint(1, len(log) / 2) + breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) + logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def json(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def jsonMultiline(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + log = f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}' + breakLineIdx1 = random.randint(1, len(log) / 2) + breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) + logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regex(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regexGBK(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}' + logger.info(str(log.encode('gbk'))) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regexMultiline(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}') + + if args.interval > 0: + time.sleep(args.interval / 1000) + +def get_random_level(): + return random.choice(['DEBUG', 'INFO', 'WARNING', 'ERROR']) + +def get_random_mark(): + return random.choice(['-', 'F']) + +def parse_custom_arg_to_dict(custom_arg): + custom_arg_dict = {} + for arg in custom_arg: + key, value = arg.split('=') + custom_arg_dict[key] = value + return custom_arg_dict + +def main(): + parser = argparse.ArgumentParser(description='Log Generator Arg Parser') + parser.add_argument('--mode', type=str, default='regex', help='Log Type') + parser.add_argument('--path', type=str, default='default.log', help='Log Path') + parser.add_argument('--count', type=int, default=100, help='Log Count') + parser.add_argument('--interval', type=int, default=1, help='Log Interval (ms), < 0 means no interval') + parser.add_argument('--custom', nargs='*', type=parse_custom_arg_to_dict, help='Custom Args, in the format of key=value') + + args = parser.parse_args() + + logger = logging.getLogger('log_generator') + logger.setLevel(logging.INFO) + # 快速轮转来模拟比较极端的情况 + handler = TimedRotatingFileHandler(args.path, when="s", interval=5, backupCount=3) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + # 随机生成器 + faker = Faker(['en_US', 'zh_CN']) + faker.add_provider(internet) + faker.add_provider(user_agent) + faker.add_provider(lorem) + faker.add_provider(misc) + + # 生成数据 + if args.mode == 'apsara': + apsara(args, logger, faker) + elif args.mode == 'delimiter': + delimiter(args, logger, faker) + elif args.mode == 'delimiterMultiline': + delimiterMultiline(args, logger, faker) + elif args.mode == 'json': + json(args, logger, faker) + elif args.mode == 'jsonMultiline': + jsonMultiline(args, logger, faker) + elif args.mode == 'regex': + regex(args, logger, faker) + elif args.mode == 'regexGBK': + regexGBK(args, logger, faker) + elif args.mode == 'regexMultiline': + regexMultiline(args, logger, faker) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/remote/log_benchmark.py b/test/engine/trigger/remote/log_benchmark.py new file mode 100644 index 0000000000..eabcc3fd77 --- /dev/null +++ b/test/engine/trigger/remote/log_benchmark.py @@ -0,0 +1,58 @@ +import argparse +import logging +import random +import time + +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime +from faker import Faker +from faker.providers import internet, user_agent, lorem, misc +def nginx(args, logger, faker): + exampleLog = f'{faker.ipv4()} - - [{datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z')}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"' + sleepInterval = len(exampleLog) / args.rate * 1024 * 1024 + startTime = datetime.now() + while True: + now = datetime.now() + logger.info(f'{faker.ipv4()} - - [{now.strftime('%d/%b/%Y:%H:%M:%S %z')}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"') + time.sleep(sleepInterval) + if (now - startTime).seconds > args.duration: + break +def parse_custom_arg_to_dict(custom_arg): + custom_arg_dict = {} + for arg in custom_arg: + key, value = arg.split('=') + custom_arg_dict[key] = value + return custom_arg_dict + +def main(): + parser = argparse.ArgumentParser(description='Log Generator Arg Parser') + parser.add_argument('--mode', type=str, default='nginx', help='Log Type') + parser.add_argument('--path', type=str, default='default.log', help='Log Path') + parser.add_argument('--rate', type=int, default=10, help='Log Generate Rate (MB/s)') + parser.add_argument('--duration', type=int, default=60, help='Log Generate Duration (s)') + parser.add_argument('--custom', nargs='*', type=parse_custom_arg_to_dict, help='Custom Args, in the format of key=value') + + args = parser.parse_args() + + logger = logging.getLogger('log_generator') + logger.setLevel(logging.INFO) + # 快速轮转来模拟比较极端的情况 + handler = TimedRotatingFileHandler(args.path, when="s", interval=5, backupCount=3) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + # 随机生成器 + faker = Faker(['en_US', 'zh_CN']) + faker.add_provider(internet) + faker.add_provider(user_agent) + faker.add_provider(lorem) + faker.add_provider(misc) + + # 生成数据 + if args.mode == 'nginx': + nginx(args, logger, faker) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/remote/mmap.py b/test/engine/trigger/remote/mmap.py new file mode 100644 index 0000000000..1a1efc5bd8 --- /dev/null +++ b/test/engine/trigger/remote/mmap.py @@ -0,0 +1,22 @@ +import argparse +import mmap +import os + +def main(): + parser = argparse.ArgumentParser(description='mmap') + parser.add_argument('--commandCnt', type=int, default=10, help='command count') + parser.add_argument('--filename', type=str, default='/tmp/loongcollector/ebpfFileSecurityHook3.log', help='filename') + + args = parser.parse_args() + + with open(args.filename, 'w') as f: + fd = f.fileno() + for i in range(args.commandCnt): + mm = mmap.mmap(fd, 20, prot=mmap.PROT_READ | mmap.PROT_WRITE, flags=mmap.MAP_SHARED) + mm.close() + + os.remove(args.filename) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/trigger.go b/test/engine/trigger/trigger.go deleted file mode 100644 index 63653c5cb6..0000000000 --- a/test/engine/trigger/trigger.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package trigger - -import ( - "context" - "strings" - "text/template" - "time" - - "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/setup" -) - -const triggerTemplate = "cd {{.WorkDir}} && TOTAL_LOG={{.TotalLog}} INTERVAL={{.Interval}} FILENAME={{.Filename}} GENERATED_LOG_DIR={{.GeneratedLogDir}} {{.Custom}} {{.Command}}" - -func RegexSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogSingle") -} - -func RegexSingleGBK(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogSingleGBK") -} - -func RegexMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogMultiline") -} - -func JSONSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJSONSingle") -} - -func JSONMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJSONMultiline") -} - -func Apsara(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateApsara") -} - -func DelimiterSingle(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateDelimiterSingle", "Delimiter", delimiter, "Quote", quote) -} - -func DelimiterMultiline(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateDelimiterMultiline", "Delimiter", delimiter, "Quote", quote) -} - -func generate(ctx context.Context, totalLog int, path string, interval int, commandName string, customKV ...string) (context.Context, error) { - time.Sleep(3 * time.Second) - command := getRunTriggerCommand(commandName) - var triggerCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerTemplate)) - splittedPath := strings.Split(path, "/") - dir := strings.Join(splittedPath[:len(splittedPath)-1], "/") - filename := splittedPath[len(splittedPath)-1] - customString := strings.Builder{} - for i := 0; i < len(customKV); i++ { - customString.WriteString(customKV[i]) - customString.WriteString("=") - customString.WriteString(customKV[i+1]) - customString.WriteString(" ") - i++ - } - if err := template.Execute(&triggerCommand, map[string]interface{}{ - "WorkDir": config.TestConfig.WorkDir, - "TotalLog": totalLog, - "Interval": interval, - "GeneratedLogDir": dir, - "Filename": filename, - "Custom": customString.String(), - "Command": command, - }); err != nil { - return ctx, err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerCommand.String()); err != nil { - return ctx, err - } - return ctx, nil -} - -func BeginTrigger(ctx context.Context) (context.Context, error) { - startTime := time.Now().Unix() - return context.WithValue(ctx, config.StartTimeContextKey, int32(startTime)), nil -} diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000000..ea45cd03b7 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1 @@ +Faker \ No newline at end of file From 25b3c7361c7436d536a3fec4f2bf96a0c60d9394 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 6 Nov 2024 15:39:06 +0800 Subject: [PATCH 2/9] fix --- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- .../case.feature | 2 +- test/config/config.go | 4 ++- test/engine/setup/docker_compose.go | 30 ++++--------------- test/engine/trigger/helper.go | 4 +-- test/engine/trigger/remote/log.py | 18 +++++------ test/engine/trigger/remote/log_benchmark.py | 15 ++++++---- 10 files changed, 33 insertions(+), 48 deletions(-) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index b8bbf86d1a..58e2ec8234 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_filebeat/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 449511f10d..43de7c8c04 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_fluentbit/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 92bb93f5d6..67e7913180 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_ilogtail/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature index 327b8d27a1..0a2cc6403f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtailspl} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_ilogtailspl/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index b0e54b85c1..334b2b3cbb 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_vector/a.log} When wait monitor until log processing finished diff --git a/test/config/config.go b/test/config/config.go index 76c9df0e73..0ddd20c517 100644 --- a/test/config/config.go +++ b/test/config/config.go @@ -15,6 +15,7 @@ package config import ( "os" + "path/filepath" "strconv" "time" @@ -73,7 +74,8 @@ func ParseConfig() { } TestConfig.WorkDir = os.Getenv("WORK_DIR") if len(TestConfig.WorkDir) == 0 { - TestConfig.WorkDir, _ = os.Getwd() + testFileDir, _ := os.Getwd() + TestConfig.WorkDir = filepath.Dir(testFileDir) } // SSH diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 49dc7538ca..3a78dfb710 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -130,29 +130,9 @@ func (d *DockerComposeEnv) ExecOnLogtail(command string) (string, error) { func (d *DockerComposeEnv) ExecOnSource(ctx context.Context, command string) (string, error) { // exec on host of docker compose - cmd := exec.Command(command) - stdout, err := cmd.StdoutPipe() - if err != nil { - return "", err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return "", err - } - if err := cmd.Start(); err != nil { - return "", err - } - buf := make([]byte, 1024) - n, _ := stdout.Read(buf) - if n > 0 { - return string(buf[:n]), nil - } - n, _ = stderr.Read(buf) - if n > 0 { - return string(buf[:n]), nil - } - if err := cmd.Wait(); err != nil { - return "", err - } - return "", nil + fmt.Println(command) + cmd := exec.Command("sh", "-c", command) + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + return string(output), err } diff --git a/test/engine/trigger/helper.go b/test/engine/trigger/helper.go index c8af4a3736..f7d9440844 100644 --- a/test/engine/trigger/helper.go +++ b/test/engine/trigger/helper.go @@ -23,7 +23,7 @@ import ( "github.com/alibaba/ilogtail/test/config" ) -const commandTemplate = "cd %s && python3 %s.py %s" +const commandTemplate = "python3 %s.py %s" func BeginTrigger(ctx context.Context) (context.Context, error) { startTime := time.Now().Unix() @@ -35,5 +35,5 @@ func GetRunTriggerCommand(triggerName string, kvs ...string) string { for i := 0; i < len(kvs); i += 2 { args = append(args, fmt.Sprintf("--%s", kvs[i]), kvs[i+1]) } - return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "test", "engine", "trigger", "remote"), triggerName, strings.Join(args, " ")) + return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "engine", "trigger", "remote", triggerName), strings.Join(args, " ")) } diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/remote/log.py index 0490240983..7351ef26f2 100644 --- a/test/engine/trigger/remote/log.py +++ b/test/engine/trigger/remote/log.py @@ -12,7 +12,7 @@ def apsara(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'[{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') + logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') if args.interval > 0: time.sleep(args.interval / 1000) @@ -22,7 +22,7 @@ def delimiter(args, logger, faker): delimiter = custom_args.get('delimiter', ' ') fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}') + logger.info(f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.http_status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}') if args.interval > 0: time.sleep(args.interval / 1000) @@ -32,7 +32,7 @@ def delimiterMultiline(args, logger, faker): delimiter = custom_args.get('delimiter', ' ') fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}' + log = f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.http_status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}' breakLineIdx1 = random.randint(1, len(log) / 2) breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) @@ -42,14 +42,14 @@ def delimiterMultiline(args, logger, faker): def json(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}') + logger.info(f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}') if args.interval > 0: time.sleep(args.interval / 1000) def jsonMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}' + log = f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}' breakLineIdx1 = random.randint(1, len(log) / 2) breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) @@ -59,14 +59,14 @@ def jsonMultiline(args, logger, faker): def regex(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}') + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}') if args.interval > 0: time.sleep(args.interval / 1000) def regexGBK(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}' + log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}' logger.info(str(log.encode('gbk'))) if args.interval > 0: time.sleep(args.interval / 1000) @@ -74,7 +74,7 @@ def regexGBK(args, logger, faker): def regexMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}') + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}') if args.interval > 0: time.sleep(args.interval / 1000) @@ -111,7 +111,7 @@ def main(): logger.addHandler(handler) # 随机生成器 - faker = Faker(['en_US', 'zh_CN']) + faker = Faker() faker.add_provider(internet) faker.add_provider(user_agent) faker.add_provider(lorem) diff --git a/test/engine/trigger/remote/log_benchmark.py b/test/engine/trigger/remote/log_benchmark.py index eabcc3fd77..98901adab0 100644 --- a/test/engine/trigger/remote/log_benchmark.py +++ b/test/engine/trigger/remote/log_benchmark.py @@ -7,16 +7,19 @@ from datetime import datetime from faker import Faker from faker.providers import internet, user_agent, lorem, misc + + def nginx(args, logger, faker): - exampleLog = f'{faker.ipv4()} - - [{datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z')}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"' - sleepInterval = len(exampleLog) / args.rate * 1024 * 1024 + exampleLog = f'{faker.ipv4()} - - [{datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"' + sleepInterval = len(exampleLog) / (args.rate * 1024 * 1024) startTime = datetime.now() while True: now = datetime.now() - logger.info(f'{faker.ipv4()} - - [{now.strftime('%d/%b/%Y:%H:%M:%S %z')}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"') + logger.info(f'{faker.ipv4()} - - [{now.strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"') time.sleep(sleepInterval) - if (now - startTime).seconds > args.duration: + if (now - startTime).seconds > args.duration * 60: break + def parse_custom_arg_to_dict(custom_arg): custom_arg_dict = {} for arg in custom_arg: @@ -29,7 +32,7 @@ def main(): parser.add_argument('--mode', type=str, default='nginx', help='Log Type') parser.add_argument('--path', type=str, default='default.log', help='Log Path') parser.add_argument('--rate', type=int, default=10, help='Log Generate Rate (MB/s)') - parser.add_argument('--duration', type=int, default=60, help='Log Generate Duration (s)') + parser.add_argument('--duration', type=int, default=60, help='Log Generate Duration (min)') parser.add_argument('--custom', nargs='*', type=parse_custom_arg_to_dict, help='Custom Args, in the format of key=value') args = parser.parse_args() @@ -43,7 +46,7 @@ def main(): logger.addHandler(handler) # 随机生成器 - faker = Faker(['en_US', 'zh_CN']) + faker = Faker() faker.add_provider(internet) faker.add_provider(user_agent) faker.add_provider(lorem) From c8a8efb0bc1ef3167cbb8a1509e1c7d817a12fac Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 6 Nov 2024 19:10:34 +0800 Subject: [PATCH 3/9] fix --- test/engine/trigger/remote/log.py | 2 +- test/engine/trigger/remote/log_benchmark.py | 22 ++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/remote/log.py index 7351ef26f2..eea7686ae9 100644 --- a/test/engine/trigger/remote/log.py +++ b/test/engine/trigger/remote/log.py @@ -74,7 +74,7 @@ def regexGBK(args, logger, faker): def regexMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}') + logger.info(f"""{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}""") if args.interval > 0: time.sleep(args.interval / 1000) diff --git a/test/engine/trigger/remote/log_benchmark.py b/test/engine/trigger/remote/log_benchmark.py index 98901adab0..7def28301b 100644 --- a/test/engine/trigger/remote/log_benchmark.py +++ b/test/engine/trigger/remote/log_benchmark.py @@ -1,5 +1,6 @@ import argparse import logging +import math import random import time @@ -8,15 +9,26 @@ from faker import Faker from faker.providers import internet, user_agent, lorem, misc +BATCH_SIZE = 100 def nginx(args, logger, faker): - exampleLog = f'{faker.ipv4()} - - [{datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"' - sleepInterval = len(exampleLog) / (args.rate * 1024 * 1024) + startTime = time.perf_counter() + exampleLog = '' + for _ in range(BATCH_SIZE): + exampleLog += f'{faker.ipv4()} - - [{datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}\n"' + randomLogCost = (time.perf_counter() - startTime) / BATCH_SIZE + writeTimePerSecond = math.floor(args.rate * 1024 * 1024 / (len(exampleLog.encode('utf-8')))) + sleepInterval = 1 / writeTimePerSecond - randomLogCost + startTime = datetime.now() while True: now = datetime.now() - logger.info(f'{faker.ipv4()} - - [{now.strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"') - time.sleep(sleepInterval) + fakeLog = f'{faker.ipv4()} - - [{now.strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"\n' * BATCH_SIZE + logger.info(fakeLog[:-1]) + if sleepInterval > 0: + start = time.perf_counter() + while (time.perf_counter() - start) < sleepInterval: + pass if (now - startTime).seconds > args.duration * 60: break @@ -40,7 +52,7 @@ def main(): logger = logging.getLogger('log_generator') logger.setLevel(logging.INFO) # 快速轮转来模拟比较极端的情况 - handler = TimedRotatingFileHandler(args.path, when="s", interval=5, backupCount=3) + handler = TimedRotatingFileHandler(args.path, when="s", interval=70, backupCount=3) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logger.addHandler(handler) From ad6241f61439d6c53d9c23e889c4070c449413e4 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 6 Nov 2024 20:04:33 +0800 Subject: [PATCH 4/9] fix --- test/engine/trigger/remote/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/remote/log.py index eea7686ae9..187a99ec20 100644 --- a/test/engine/trigger/remote/log.py +++ b/test/engine/trigger/remote/log.py @@ -74,7 +74,8 @@ def regexGBK(args, logger, faker): def regexMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f"""{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {'\n'.join(faker.sentences(nb=random.randint(1, 5)))}""") + multilineLog = '\n'.join(faker.sentences(nb=random.randint(1, 5))) + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {multilineLog}') if args.interval > 0: time.sleep(args.interval / 1000) From b01c366fe22220c80cf1b20bb6e68c9358d94f08 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 7 Nov 2024 00:07:39 +0800 Subject: [PATCH 5/9] fix --- test/engine/trigger/remote/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/remote/log.py index 187a99ec20..549c7530f1 100644 --- a/test/engine/trigger/remote/log.py +++ b/test/engine/trigger/remote/log.py @@ -59,14 +59,14 @@ def jsonMultiline(args, logger, faker): def regex(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}') + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.user_agent()}" "{faker.sentence()}"') if args.interval > 0: time.sleep(args.interval / 1000) def regexGBK(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} {faker.sentence()}' + log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.user_agent()}" "{faker.sentence()}"' logger.info(str(log.encode('gbk'))) if args.interval > 0: time.sleep(args.interval / 1000) @@ -75,7 +75,7 @@ def regexMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): multilineLog = '\n'.join(faker.sentences(nb=random.randint(1, 5))) - logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} {faker.user_agent()} java.lang.Exception: {multilineLog}') + logger.info(f'{get_random_mark()} file{fileNo}:{i} [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] [{get_random_level()}] java.lang.Exception: {multilineLog}') if args.interval > 0: time.sleep(args.interval / 1000) From 7534b8b6f215c09df5f3f39abcf2aa78df2fdfc3 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 7 Nov 2024 14:24:38 +0800 Subject: [PATCH 6/9] fix --- .../trigger/{remote/mmap.py => ebpf/remote_mmap.py} | 0 test/engine/trigger/ebpf/security.go | 2 +- test/engine/trigger/helper.go | 4 ++-- test/engine/trigger/log/file.go | 13 ++++++++----- test/engine/trigger/log/{http.go => http_server.go} | 0 .../trigger/{remote/log.py => log/remote_file.py} | 0 .../remote_file_benchmark.py} | 0 7 files changed, 11 insertions(+), 8 deletions(-) rename test/engine/trigger/{remote/mmap.py => ebpf/remote_mmap.py} (100%) rename test/engine/trigger/log/{http.go => http_server.go} (100%) rename test/engine/trigger/{remote/log.py => log/remote_file.py} (100%) rename test/engine/trigger/{remote/log_benchmark.py => log/remote_file_benchmark.py} (100%) diff --git a/test/engine/trigger/remote/mmap.py b/test/engine/trigger/ebpf/remote_mmap.py similarity index 100% rename from test/engine/trigger/remote/mmap.py rename to test/engine/trigger/ebpf/remote_mmap.py diff --git a/test/engine/trigger/ebpf/security.go b/test/engine/trigger/ebpf/security.go index 9cc85d527b..34bfc189c3 100644 --- a/test/engine/trigger/ebpf/security.go +++ b/test/engine/trigger/ebpf/security.go @@ -113,7 +113,7 @@ func rwFile(ctx context.Context, commandCnt int, filenames string) error { func mmapFile(ctx context.Context, commandCnt int, filenames string) error { files := strings.Split(filenames, ",") for _, file := range files { - mmapFileCommand := trigger.GetRunTriggerCommand("mmap", "commandCnt", strconv.FormatInt(int64(commandCnt), 10), "filename", file) + mmapFileCommand := trigger.GetRunTriggerCommand("ebpf", "mmap", "commandCnt", strconv.FormatInt(int64(commandCnt), 10), "filename", file) if _, err := setup.Env.ExecOnSource(ctx, mmapFileCommand); err != nil { return err } diff --git a/test/engine/trigger/helper.go b/test/engine/trigger/helper.go index f7d9440844..696c63d0b2 100644 --- a/test/engine/trigger/helper.go +++ b/test/engine/trigger/helper.go @@ -30,10 +30,10 @@ func BeginTrigger(ctx context.Context) (context.Context, error) { return context.WithValue(ctx, config.StartTimeContextKey, int32(startTime)), nil } -func GetRunTriggerCommand(triggerName string, kvs ...string) string { +func GetRunTriggerCommand(scenrio, triggerName string, kvs ...string) string { args := make([]string, 0) for i := 0; i < len(kvs); i += 2 { args = append(args, fmt.Sprintf("--%s", kvs[i]), kvs[i+1]) } - return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "engine", "trigger", "remote", triggerName), strings.Join(args, " ")) + return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "engine", "trigger", scenrio, "remote_"+triggerName), strings.Join(args, " ")) } diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go index dd1275e4bd..812812da3f 100644 --- a/test/engine/trigger/log/file.go +++ b/test/engine/trigger/log/file.go @@ -15,6 +15,7 @@ package log import ( "context" + "fmt" "strconv" "time" @@ -61,17 +62,19 @@ func Nginx(ctx context.Context, rate, duration int, path string) (context.Contex func generate(ctx context.Context, mode, path string, count, interval int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) customKV = append(customKV, "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval)) - command := trigger.GetRunTriggerCommand("log", customKV...) - if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { - return ctx, err - } + command := trigger.GetRunTriggerCommand("log", "file", customKV...) + go func() { + if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { + fmt.Println(err) + } + }() return ctx, nil } func generateBenchmark(ctx context.Context, mode, path string, rate, duration int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) customKV = append(customKV, "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration)) - command := trigger.GetRunTriggerCommand("log_benchmark", customKV...) + command := trigger.GetRunTriggerCommand("log", "file_benchmark", customKV...) if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { return ctx, err } diff --git a/test/engine/trigger/log/http.go b/test/engine/trigger/log/http_server.go similarity index 100% rename from test/engine/trigger/log/http.go rename to test/engine/trigger/log/http_server.go diff --git a/test/engine/trigger/remote/log.py b/test/engine/trigger/log/remote_file.py similarity index 100% rename from test/engine/trigger/remote/log.py rename to test/engine/trigger/log/remote_file.py diff --git a/test/engine/trigger/remote/log_benchmark.py b/test/engine/trigger/log/remote_file_benchmark.py similarity index 100% rename from test/engine/trigger/remote/log_benchmark.py rename to test/engine/trigger/log/remote_file_benchmark.py From b0c9c76ff9ddfdab78a96df0ab4002e03a834500 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 7 Nov 2024 17:46:40 +0800 Subject: [PATCH 7/9] fix --- core/unittest/pipeline/PipelineUnittest.cpp | 1 + test/engine/trigger/log/file.go | 15 +++++++++++---- test/engine/trigger/log/remote_file.py | 6 +++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 1cf92420d5..df709defa1 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2916,6 +2916,7 @@ void PipelineUnittest::TestWaitAllItemsInProcessFinished() const { APSARA_TEST_NOT_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); pipeline->mInProcessCnt.store(0); // recover + usleep(1000); APSARA_TEST_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); } diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go index 812812da3f..0d7a767854 100644 --- a/test/engine/trigger/log/file.go +++ b/test/engine/trigger/log/file.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/alibaba/ilogtail/test/engine/setup" @@ -61,8 +62,11 @@ func Nginx(ctx context.Context, rate, duration int, path string) (context.Contex func generate(ctx context.Context, mode, path string, count, interval int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) - customKV = append(customKV, "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval)) - command := trigger.GetRunTriggerCommand("log", "file", customKV...) + customKVString := make([]string, 0) + for i := 0; i < len(customKV); i += 2 { + customKVString = append(customKVString, customKV[i]+"="+customKV[i+1]) + } + command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", strings.Join(customKVString, " ")) go func() { if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { fmt.Println(err) @@ -73,8 +77,11 @@ func generate(ctx context.Context, mode, path string, count, interval int, custo func generateBenchmark(ctx context.Context, mode, path string, rate, duration int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) - customKV = append(customKV, "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration)) - command := trigger.GetRunTriggerCommand("log", "file_benchmark", customKV...) + customKVString := make([]string, 0) + for i := 0; i < len(customKV); i += 2 { + customKVString = append(customKVString, customKV[i]+"="+customKV[i+1]) + } + command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", strings.Join(customKVString, " ")) if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { return ctx, err } diff --git a/test/engine/trigger/log/remote_file.py b/test/engine/trigger/log/remote_file.py index 549c7530f1..e6a6c053d0 100644 --- a/test/engine/trigger/log/remote_file.py +++ b/test/engine/trigger/log/remote_file.py @@ -12,7 +12,7 @@ def apsara(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') + logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}:{i}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') if args.interval > 0: time.sleep(args.interval / 1000) @@ -50,8 +50,8 @@ def jsonMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): log = f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}' - breakLineIdx1 = random.randint(1, len(log) / 2) - breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) + breakLineIdx1 = random.randint(1, len(log) // 2) + breakLineIdx2 = random.randint(len(log) // 2 + 1, len(log) - 1) logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) if args.interval > 0: time.sleep(args.interval / 1000) From deccab64766ee51e810b98d4ee38cd413100b890 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 8 Nov 2024 00:47:45 +0800 Subject: [PATCH 8/9] fix --- test/engine/trigger/log/file.go | 22 +++++++++++++------ test/engine/trigger/log/remote_file.py | 11 ++++------ .../trigger/log/remote_file_benchmark.py | 7 ++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go index 0d7a767854..0f538bd0b3 100644 --- a/test/engine/trigger/log/file.go +++ b/test/engine/trigger/log/file.go @@ -15,9 +15,9 @@ package log import ( "context" + "encoding/json" "fmt" "strconv" - "strings" "time" "github.com/alibaba/ilogtail/test/engine/setup" @@ -62,11 +62,15 @@ func Nginx(ctx context.Context, rate, duration int, path string) (context.Contex func generate(ctx context.Context, mode, path string, count, interval int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) - customKVString := make([]string, 0) + customKVString := make(map[string]string) for i := 0; i < len(customKV); i += 2 { - customKVString = append(customKVString, customKV[i]+"="+customKV[i+1]) + customKVString[customKV[i]] = customKV[i+1] } - command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", strings.Join(customKVString, " ")) + jsonStr, err := json.Marshal(customKVString) + if err != nil { + return ctx, err + } + command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", string(jsonStr)) go func() { if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { fmt.Println(err) @@ -77,11 +81,15 @@ func generate(ctx context.Context, mode, path string, count, interval int, custo func generateBenchmark(ctx context.Context, mode, path string, rate, duration int, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) - customKVString := make([]string, 0) + customKVString := make(map[string]string) for i := 0; i < len(customKV); i += 2 { - customKVString = append(customKVString, customKV[i]+"="+customKV[i+1]) + customKVString[customKV[i]] = customKV[i+1] + } + jsonStr, err := json.Marshal(customKVString) + if err != nil { + return ctx, err } - command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", strings.Join(customKVString, " ")) + command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", string(jsonStr)) if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { return ctx, err } diff --git a/test/engine/trigger/log/remote_file.py b/test/engine/trigger/log/remote_file.py index e6a6c053d0..b803fb6644 100644 --- a/test/engine/trigger/log/remote_file.py +++ b/test/engine/trigger/log/remote_file.py @@ -1,4 +1,5 @@ import argparse +import json as jsonlib import logging import random import time @@ -12,7 +13,7 @@ def apsara(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[{faker.uri_path()}:{i}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') + logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[/build/core/application/Application:{i}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') if args.interval > 0: time.sleep(args.interval / 1000) @@ -87,11 +88,7 @@ def get_random_mark(): return random.choice(['-', 'F']) def parse_custom_arg_to_dict(custom_arg): - custom_arg_dict = {} - for arg in custom_arg: - key, value = arg.split('=') - custom_arg_dict[key] = value - return custom_arg_dict + return jsonlib.loads(custom_arg) def main(): parser = argparse.ArgumentParser(description='Log Generator Arg Parser') @@ -99,7 +96,7 @@ def main(): parser.add_argument('--path', type=str, default='default.log', help='Log Path') parser.add_argument('--count', type=int, default=100, help='Log Count') parser.add_argument('--interval', type=int, default=1, help='Log Interval (ms), < 0 means no interval') - parser.add_argument('--custom', nargs='*', type=parse_custom_arg_to_dict, help='Custom Args, in the format of key=value') + parser.add_argument('--custom', type=parse_custom_arg_to_dict, help='Custom Args, in the format of json') args = parser.parse_args() diff --git a/test/engine/trigger/log/remote_file_benchmark.py b/test/engine/trigger/log/remote_file_benchmark.py index 7def28301b..b9f53e00da 100644 --- a/test/engine/trigger/log/remote_file_benchmark.py +++ b/test/engine/trigger/log/remote_file_benchmark.py @@ -1,4 +1,5 @@ import argparse +import json import logging import math import random @@ -33,11 +34,7 @@ def nginx(args, logger, faker): break def parse_custom_arg_to_dict(custom_arg): - custom_arg_dict = {} - for arg in custom_arg: - key, value = arg.split('=') - custom_arg_dict[key] = value - return custom_arg_dict + return json.loads(custom_arg) def main(): parser = argparse.ArgumentParser(description='Log Generator Arg Parser') From 1a4db5a7ad2dbdcb1abaf5560056b4b106a60aa4 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 8 Nov 2024 10:52:49 +0800 Subject: [PATCH 9/9] fix --- core/unittest/pipeline/PipelineUnittest.cpp | 2 +- test/engine/trigger/log/file.go | 12 ++++- test/engine/trigger/log/remote_file.py | 52 +++++++++++++++++---- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index df709defa1..396e84627c 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2916,7 +2916,7 @@ void PipelineUnittest::TestWaitAllItemsInProcessFinished() const { APSARA_TEST_NOT_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); pipeline->mInProcessCnt.store(0); // recover - usleep(1000); + usleep(3000); APSARA_TEST_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); } diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go index 0f538bd0b3..ddea428021 100644 --- a/test/engine/trigger/log/file.go +++ b/test/engine/trigger/log/file.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "time" "github.com/alibaba/ilogtail/test/engine/setup" @@ -70,7 +71,8 @@ func generate(ctx context.Context, mode, path string, count, interval int, custo if err != nil { return ctx, err } - command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", string(jsonStr)) + command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", wrapperCustomArgs(string(jsonStr))) + fmt.Println(command) go func() { if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { fmt.Println(err) @@ -89,9 +91,15 @@ func generateBenchmark(ctx context.Context, mode, path string, rate, duration in if err != nil { return ctx, err } - command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", string(jsonStr)) + command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", wrapperCustomArgs(string(jsonStr))) if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { return ctx, err } return ctx, nil } + +func wrapperCustomArgs(customArgs string) string { + fmt.Println(customArgs) + customArgs = strings.ReplaceAll(customArgs, "\\", "\\\\") + return "\"" + strings.ReplaceAll(customArgs, "\"", "\\\"") + "\"" +} diff --git a/test/engine/trigger/log/remote_file.py b/test/engine/trigger/log/remote_file.py index b803fb6644..1415c48e36 100644 --- a/test/engine/trigger/log/remote_file.py +++ b/test/engine/trigger/log/remote_file.py @@ -23,7 +23,21 @@ def delimiter(args, logger, faker): delimiter = custom_args.get('delimiter', ' ') fileNo = random.randint(1, 1000) for i in range(args.count): - logger.info(f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.http_status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}') + logParts = [ + f'{quote}{get_random_mark()}{quote}', + f'{quote}file{fileNo}{quote}', + f'{quote}logNo:{i}{quote}', + f'{quote}{faker.ipv4()}{quote}', + f'{quote}{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}{quote}', + f'{quote}{faker.http_method()}{quote}', + f'{quote}{faker.uri_path()}{quote}', + f'{quote}HTTP/2.0{quote}', + f'{quote}{faker.http_status_code()}{quote}', + f'{quote}{random.randint(1, 10000)}{quote}', + f'{quote}{faker.user_agent()}{quote}' + ] + log = delimiter.join(logParts) + logger.info(log) if args.interval > 0: time.sleep(args.interval / 1000) @@ -33,10 +47,21 @@ def delimiterMultiline(args, logger, faker): delimiter = custom_args.get('delimiter', ' ') fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{quote}{get_random_mark()}{quote}{delimiter}{quote}file{fileNo}{quote}{delimiter}{quote}logNo:{i}{quote}{delimiter}{quote}{faker.ipv4()}{quote}{delimiter}{quote}{faker.http_method()}{quote}{delimiter}{quote}{faker.uri_path()}{quote}{delimiter}{quote}HTTP/2.0{quote}{delimiter}{quote}{faker.http_status_code()}{quote}{delimiter}{quote}{faker.user_agent()}{quote}' - breakLineIdx1 = random.randint(1, len(log) / 2) - breakLineIdx2 = random.randint(len(log) / 2, len(log) - 1) - logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) + logParts = [ + f'{quote}{get_random_mark()}{quote}', + f'{quote}fi\nle{fileNo}{quote}', + f'{quote}logNo\n:{i}{quote}', + f'{quote}{faker.ipv4()}{quote}', + f'{quote}{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}{quote}', + f'{quote}{faker.http_method()}{quote}', + f'{quote}{faker.uri_path()}{quote}', + f'{quote}HT\nTP/2.0{quote}', + f'{quote}{faker.http_status_code()}{quote}', + f'{quote}{random.randint(1, 10000)}{quote}', + f'{quote}{faker.user_agent()}{quote}' + ] + log = delimiter.join(logParts) + logger.info(log) if args.interval > 0: time.sleep(args.interval / 1000) @@ -50,10 +75,18 @@ def json(args, logger, faker): def jsonMultiline(args, logger, faker): fileNo = random.randint(1, 1000) for i in range(args.count): - log = f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}' - breakLineIdx1 = random.randint(1, len(log) // 2) - breakLineIdx2 = random.randint(len(log) // 2 + 1, len(log) - 1) - logger.info(log[:breakLineIdx1] + '\n' + log[breakLineIdx1:breakLineIdx2] + '\n' + log[breakLineIdx2:]) + logParts = [ + f'"mark":"{get_random_mark()}"', + f'"file":"file{fileNo}"', + f'"logNo":{i}', + f'"time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}"', + f'"ip": "{faker.ipv4()}"', + f'"method": "{faker.http_method()}"', + f'"userAgent": "{faker.user_agent()}"', + f'"size": {random.randint(1, 10000)}' + ] + log = '{' + ',\n'.join(logParts) + '}' + logger.info(log) if args.interval > 0: time.sleep(args.interval / 1000) @@ -106,6 +139,7 @@ def main(): handler = TimedRotatingFileHandler(args.path, when="s", interval=5, backupCount=3) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) + handler.flush = lambda: handler.stream.flush() logger.addHandler(handler) # 随机生成器