diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 5cc0ff4411..2f29670fca 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -37,6 +37,7 @@ jobs: strategy: matrix: go-version: [ 1.19.10 ] + python-version: [ 3.8 ] runner: [ ubuntu-latest ] fail-fast: true permissions: @@ -62,6 +63,11 @@ jobs: with: go-version: ${{ matrix.go-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Check out code uses: actions/checkout@v2 with: @@ -83,6 +89,7 @@ jobs: BUILD_LOGTAIL_UT: OFF WITHOUTGDB: ON run: | + pip3 install -r test/requirements.txt make benchmark git stash diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 1cf92420d5..396e84627c 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2916,6 +2916,7 @@ void PipelineUnittest::TestWaitAllItemsInProcessFinished() const { APSARA_TEST_NOT_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); pipeline->mInProcessCnt.store(0); // recover + usleep(3000); APSARA_TEST_EQUAL(std::future_status::ready, future.wait_for(std::chrono::seconds(0))); } diff --git a/docker/Dockerfile.e2e-test b/docker/Dockerfile.e2e-test index 785df4caa3..e2acd47d42 100644 --- a/docker/Dockerfile.e2e-test +++ b/docker/Dockerfile.e2e-test @@ -1,12 +1,11 @@ -FROM golang:1.19 +FROM python:3.8 -RUN go env -w GOPROXY="https://goproxy.cn,direct" RUN mkdir -p /tmp/loongcollector WORKDIR /root COPY . ./loongcollector WORKDIR /root/loongcollector/test -RUN go mod download +RUN pip3 install -r requirements.txt CMD ["sh", "-c", "while true; do sleep 3600; done"] \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature index b8bbf86d1a..58e2ec8234 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole filebeat Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_filebeat} When start monitor {filebeat} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_filebeat/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature index 449511f10d..43de7c8c04 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole fluentbit Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_fluentbit} When start monitor {fluent-bit} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_fluentbit/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 92bb93f5d6..67e7913180 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_ilogtail/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature index 327b8d27a1..0a2cc6403f 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole iLogtail Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_ilogtailspl} When start monitor {ilogtailC} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_ilogtailspl/a.log} When wait monitor until log processing finished diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature index b0e54b85c1..334b2b3cbb 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -8,5 +8,5 @@ Feature: performance file to blackhole vector Given docker-compose boot type {benchmark} When start docker-compose {performance_file_to_blackhole_vector} When start monitor {vector} - When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./a.log} + When generate random nginx logs to file, speed {10}MB/s, total {3}min, to file {./test_cases/performance_file_to_blackhole_vector/a.log} When wait monitor until log processing finished diff --git a/test/config/config.go b/test/config/config.go index 915bd08c6d..0ddd20c517 100644 --- a/test/config/config.go +++ b/test/config/config.go @@ -15,6 +15,7 @@ package config import ( "os" + "path/filepath" "strconv" "time" @@ -72,6 +73,10 @@ func ParseConfig() { TestConfig.GeneratedLogDir = "/tmp/loongcollector" } TestConfig.WorkDir = os.Getenv("WORK_DIR") + if len(TestConfig.WorkDir) == 0 { + testFileDir, _ := os.Getwd() + TestConfig.WorkDir = filepath.Dir(testFileDir) + } // SSH TestConfig.SSHUsername = os.Getenv("SSH_USERNAME") diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 6a7e3fc4de..3a78dfb710 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "time" @@ -128,5 +129,10 @@ func (d *DockerComposeEnv) ExecOnLogtail(command string) (string, error) { } func (d *DockerComposeEnv) ExecOnSource(ctx context.Context, command string) (string, error) { - return "", fmt.Errorf("not implemented") + // exec on host of docker compose + fmt.Println(command) + cmd := exec.Command("sh", "-c", command) + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + return string(output), err } diff --git a/test/engine/steps.go b/test/engine/steps.go index 7b19db1988..336a5ff8ac 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -10,8 +10,11 @@ import ( "github.com/alibaba/ilogtail/test/engine/cleanup" "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" "github.com/alibaba/ilogtail/test/engine/trigger" + "github.com/alibaba/ilogtail/test/engine/trigger/ebpf" + "github.com/alibaba/ilogtail/test/engine/trigger/log" "github.com/alibaba/ilogtail/test/engine/verify" ) @@ -27,6 +30,7 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig) ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) ctx.Given(`^mkdir \{(.*)\}`, setup.Mkdir) + ctx.Given(`^docker-compose boot type \{(\S+)\}$`, setup.SetDockerComposeBootType) // ------------------------------------------ // When @@ -41,19 +45,25 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { // generate ctx.When(`^begin trigger`, trigger.BeginTrigger) - ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingle) - ctx.When(`^generate \{(\d+)\} multiline regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexMultiline) - ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.RegexSingleGBK) - ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP) - ctx.When(`^generate \{(\d+)\} apsara logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.Apsara) - ctx.When(`^generate \{(\d+)\} delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterSingle) - ctx.When(`^generate \{(\d+)\} multiline delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterMultiline) - ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONSingle) - ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONMultiline) - ctx.When(`^execute \{(\d+)\} commands to generate process security events`, trigger.TrigerProcessSecurityEvents) - ctx.When(`^execute \{(\d+)\} commands to generate network security events on url \{(.*)\}$`, trigger.TrigerNetworksSecurityEvents) - ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) - ctx.When(`^generate \{(\d+)\} HTTP requests, with interval \{(\d+)\}ms, url: \{(.*)\}`, trigger.TrigerHTTP) + // log + ctx.When(`^generate \{(\d+)\} regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexSingle) + ctx.When(`^generate \{(\d+)\} multiline regex logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexMultiline) + ctx.When(`^generate \{(\d+)\} regex gbk logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.RegexSingleGBK) + ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, log.HTTP) + ctx.When(`^generate \{(\d+)\} apsara logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.Apsara) + ctx.When(`^generate \{(\d+)\} delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, log.DelimiterSingle) + ctx.When(`^generate \{(\d+)\} multiline delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, log.DelimiterMultiline) + ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.JSONSingle) + ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, log.JSONMultiline) + ctx.When(`^generate random nginx logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}`, log.Nginx) + ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor) + ctx.When(`^wait monitor until log processing finished$`, monitor.WaitMonitorUntilProcessingFinished) + + // ebpf + ctx.When(`^execute \{(\d+)\} commands to generate process security events`, ebpf.ProcessSecurityEvents) + ctx.When(`^execute \{(\d+)\} commands to generate network security events on url \{(.*)\}$`, ebpf.NetworksSecurityEvents) + ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, ebpf.FileSecurityEvents) + ctx.When(`^generate \{(\d+)\} HTTP requests, with interval \{(\d+)\}ms, url: \{(.*)\}`, ebpf.HTTP) // ------------------------------------------ // Then diff --git a/test/engine/trigger/protocol.go b/test/engine/trigger/ebpf/http.go similarity index 90% rename from test/engine/trigger/protocol.go rename to test/engine/trigger/ebpf/http.go index 646a4aa97b..c3c84f9ddd 100644 --- a/test/engine/trigger/protocol.go +++ b/test/engine/trigger/ebpf/http.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package ebpf import ( "context" @@ -22,7 +22,7 @@ import ( "github.com/alibaba/ilogtail/test/engine/setup" ) -func TrigerHTTP(ctx context.Context, count int, interval int, url string) (context.Context, error) { +func HTTP(ctx context.Context, count int, interval int, url string) (context.Context, error) { logger.Debugf(context.Background(), "count:%d interval:%d url:%s", count, interval, url) cmd := fmt.Sprintf("curl -vL %s", url) time.Sleep(time.Second * 5) diff --git a/test/engine/trigger/ebpf/remote_mmap.py b/test/engine/trigger/ebpf/remote_mmap.py new file mode 100644 index 0000000000..1a1efc5bd8 --- /dev/null +++ b/test/engine/trigger/ebpf/remote_mmap.py @@ -0,0 +1,22 @@ +import argparse +import mmap +import os + +def main(): + parser = argparse.ArgumentParser(description='mmap') + parser.add_argument('--commandCnt', type=int, default=10, help='command count') + parser.add_argument('--filename', type=str, default='/tmp/loongcollector/ebpfFileSecurityHook3.log', help='filename') + + args = parser.parse_args() + + with open(args.filename, 'w') as f: + fd = f.fileno() + for i in range(args.commandCnt): + mm = mmap.mmap(fd, 20, prot=mmap.PROT_READ | mmap.PROT_WRITE, flags=mmap.MAP_SHARED) + mm.close() + + os.remove(args.filename) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/ebpf_trigger.go b/test/engine/trigger/ebpf/security.go similarity index 76% rename from test/engine/trigger/ebpf_trigger.go rename to test/engine/trigger/ebpf/security.go index 4b824c557e..34bfc189c3 100644 --- a/test/engine/trigger/ebpf_trigger.go +++ b/test/engine/trigger/ebpf/security.go @@ -11,16 +11,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package ebpf import ( "context" - "html/template" + "strconv" "strings" "time" - "github.com/alibaba/ilogtail/test/config" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/trigger" ) /* @@ -28,7 +28,7 @@ import ( input_process_security ******************** */ -func TrigerProcessSecurityEvents(ctx context.Context, commandCnt int) (context.Context, error) { +func ProcessSecurityEvents(ctx context.Context, commandCnt int) (context.Context, error) { time.Sleep(5 * time.Second) if err := execveCommands(ctx, commandCnt); err != nil { return ctx, err @@ -51,7 +51,7 @@ func execveCommands(ctx context.Context, commandCnt int) error { input_network_security ******************** */ -func TrigerNetworksSecurityEvents(ctx context.Context, commandCnt int, url string) (context.Context, error) { +func NetworksSecurityEvents(ctx context.Context, commandCnt int, url string) (context.Context, error) { time.Sleep(5 * time.Second) if err := curlURL(ctx, commandCnt, url); err != nil { return ctx, err @@ -74,9 +74,8 @@ func curlURL(ctx context.Context, commandCnt int, url string) error { input_file_security ******************** */ -const triggerFileSecurityTemplate = "cd {{.WorkDir}} && COMMAND_CNT={{.CommandCnt}} FILE_NAME={{.FileName}} {{.Command}}" -func TrigerFileSecurityEvents(ctx context.Context, commandCnt int, filenames string) (context.Context, error) { +func FileSecurityEvents(ctx context.Context, commandCnt int, filenames string) (context.Context, error) { time.Sleep(5 * time.Second) if err := rwFile(ctx, commandCnt, filenames); err != nil { return ctx, err @@ -112,20 +111,10 @@ func rwFile(ctx context.Context, commandCnt int, filenames string) error { } func mmapFile(ctx context.Context, commandCnt int, filenames string) error { - mmapFileCommand := getRunTriggerCommand("TestGenerateMmapCommand") files := strings.Split(filenames, ",") for _, file := range files { - var triggerEBPFCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerFileSecurityTemplate)) - if err := template.Execute(&triggerEBPFCommand, map[string]interface{}{ - "WorkDir": config.TestConfig.WorkDir, - "CommandCnt": commandCnt, - "FileName": file, - "Command": mmapFileCommand, - }); err != nil { - return err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerEBPFCommand.String()); err != nil { + mmapFileCommand := trigger.GetRunTriggerCommand("ebpf", "mmap", "commandCnt", strconv.FormatInt(int64(commandCnt), 10), "filename", file) + if _, err := setup.Env.ExecOnSource(ctx, mmapFileCommand); err != nil { return err } } diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go deleted file mode 100644 index 373aebbe52..0000000000 --- a/test/engine/trigger/file.go +++ /dev/null @@ -1,114 +0,0 @@ -package trigger - -import ( - "context" - "fmt" - "math/rand" - "os" - "path/filepath" - "time" - - "golang.org/x/time/rate" - - "github.com/alibaba/ilogtail/test/config" -) - -// JSON template -func GenerateRandomNginxLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) { - - // clear file - path = filepath.Clean(path) - path = filepath.Join(config.CaseHome, path) - fmt.Println(path) - _ = os.WriteFile(path, []byte{}, 0600) - file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304 - - rand.Seed(time.Now().UnixNano()) - maxLogLen := 1024 - nginxLog := genNginxLog() - - limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLogLen) - - timeout := time.After(time.Minute * time.Duration(totalTime)) - - for { - select { - // context is done - case <-ctx.Done(): - // clear file - _ = file.Close() - return ctx, nil - // all time is done - case <-timeout: - // clear file - _ = file.Close() - return ctx, nil - default: - if limiter.AllowN(time.Now(), len(nginxLog)) { - _, _ = file.WriteString(nginxLog + "\n") // #nosec G307 - nginxLog = genNginxLog() - } - } - } -} - -var ipAddresses = []string{ - "103.159.151.180", - "12.55.18.241", - "182.233.128.102", - "221.85.57.231", - "76.245.65.224", - "86.250.231.93", - "44.201.253.252", - "218.7.2.219", - "172.118.174.109", - "208.16.46.154", - "7.138.80.41", - "214.73.25.80", - "83.124.20.79", - "80.226.48.153", - "92.129.204.161", - "212.103.145.159", - "148.188.8.90", - "148.212.244.121", - "106.186.172.157", - "30.127.196.158", -} - -var userAgents = []string{ - "aliyun-sdk-java", - "aliyun-sdk-golang", - "aliyun-sdk-python", -} - -var statusCodes = []string{ - "400", - "401", - "402", - "403", - "404", - "200", -} - -const bytesMean = 5500.0 -const bytesStddev = 1500.0 - -func genNginxLog() string { - nginxLogTemplate := `%s - - [%s] "GET http://www.districtdot-com.biz/syndicate HTTP/1.1" %s %d "http://www.chiefscalable.biz/webservices" "%s"` - currentTime := time.Now().Format("02/Jan/2006:15:04:05 +0800") - ipAddress := ipAddresses[rand.Intn(len(ipAddresses))] // #nosec G404 - statusIdx := rand.Intn(len(statusCodes) * 10) // #nosec G404 - if statusIdx >= len(statusCodes) { - statusIdx = len(statusCodes) - 1 - } - bytesSize := int32(rand.NormFloat64()*bytesStddev + bytesMean) - if bytesSize < 1000 { - bytesSize = 0 - } else if bytesSize > 10000 { - bytesSize = 10000 - } - statusCode := statusCodes[statusIdx] - userAgent := userAgents[rand.Intn(len(userAgents))] // #nosec G404 - - return fmt.Sprintf(nginxLogTemplate, ipAddress, currentTime, statusCode, bytesSize, userAgent) -} diff --git a/test/engine/trigger/generator/apsara_test.go b/test/engine/trigger/generator/apsara_test.go deleted file mode 100644 index 40329c0cfe..0000000000 --- a/test/engine/trigger/generator/apsara_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "strconv" - "testing" - "time" -) - -// TestGenerateApsara will be executed in the environment being collected. -func TestGenerateApsara(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/application/Application:12]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:hello world!\n", - "[{{.Time}}]\t[{{.Level}}]\t[20964]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:这是一条消息\n", - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:hello world!\n", - "[{{.Time}}]\t[{{.Level}}]\t[32337]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:这是一条消息\n", - "[{{.Time}}]\t[{{.Level}}]\t[00001]\t[/build/core/ilogtail.cpp:127]\tfile:file{{.FileNo}}\tlogNo:{{.LogNo}}\tmark:{{.Mark}}\tmsg:password:123456\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - var currentTime string - if i%2 == 0 { - currentTime = time.Now().Format("2006-01-02 15:04:05.000000") - } else { - currentTime = strconv.FormatInt(time.Now().UnixNano()/1000, 10) - } - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": currentTime, - "Level": getRandomLogLevel(), - "LogNo": logNo + i, - "FileNo": fileNo, - "Mark": getRandomMark(), - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/delimiter_test.go b/test/engine/trigger/generator/delimiter_test.go deleted file mode 100644 index 354b6dea8c..0000000000 --- a/test/engine/trigger/generator/delimiter_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "testing" - "time" -) - -// TestGenerateDelimiterSingle will be executed in the environment being collected. -func TestGenerateDelimiterSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv("Delimiter", "Quote") - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - delimiter := config.Custom["Delimiter"] - if delimiter == "" { - delimiter = " " - } - quote := config.Custom["Quote"] - if quote == "" { - quote = "" - } - testLogContentTmpl := string2Template([]string{ - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}0.0.0.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/index.html{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}628{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}218{{.Quote}}{{.Delimiter}}{{.Quote}}go-sdk{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}744{{.Quote}}{{.Delimiter}}{{.Quote}}Mozilla/5.0{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}127.0.0.1{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}200{{.Quote}}{{.Delimiter}}{{.Quote}}320{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}{{.Mark}}{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}192.168.0.3{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}HTTP/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}949{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo, - "Time": time.Now().Format("2006-01-02 15:04:05.000000000"), - "Delimiter": delimiter, - "Quote": quote, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -// TestGenerateDelimiterMultiline will be executed in the environment being collected. -func TestGenerateDelimiterMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv("Delimiter", "Quote") - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - delimiter := config.Custom["Delimiter"] - if delimiter == "" { - delimiter = " " - } - quote := config.Custom["Quote"] - if quote == "" { - quote = "" - } - testLogContentTmpl := string2Template([]string{ - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}0.0.0.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/index.html{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}628{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}-{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}302{{.Quote}}{{.Delimiter}}{{.Quote}}218{{.Quote}}{{.Delimiter}}{{.Quote}}go-sdk{{.Quote}}\n", - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}10.45.26.0{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}GET{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}744{{.Quote}}{{.Delimiter}}{{.Quote}}Mozilla/5.0{{.Quote}}\n", - "{{.Quote}}-{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}127.0.0.1{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/2.0{{.Quote}}{{.Delimiter}}{{.Quote}}200{{.Quote}}{{.Delimiter}}{{.Quote}}320{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - "{{.Quote}}F{{.Quote}}{{.Delimiter}}{{.Quote}}file{{.FileNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}{{.LogNo}}{{.Quote}}{{.Delimiter}}{{.Quote}}192.168.0.3{{.Quote}}{{.Delimiter}}{{.Quote}}{{.Time}}{{.Quote}}{{.Delimiter}}{{.Quote}}PUT{{.Quote}}{{.Delimiter}}{{.Quote}}/dir/resource.txt{{.Quote}}{{.Delimiter}}{{.Quote}}\nHTTP\n/1.1{{.Quote}}{{.Delimiter}}{{.Quote}}404{{.Quote}}{{.Delimiter}}{{.Quote}}949{{.Quote}}{{.Delimiter}}{{.Quote}}curl/7.10{{.Quote}}\n", - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "FileNo": fileNo, - "LogNo": logNo, - "Time": time.Now().Format("2006-01-02 15:04:05.000000000"), - "Delimiter": delimiter, - "Quote": quote, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/ebpf_file_mmap_test.go b/test/engine/trigger/generator/ebpf_file_mmap_test.go deleted file mode 100644 index 56e221b54d..0000000000 --- a/test/engine/trigger/generator/ebpf_file_mmap_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "os" - "strconv" - "syscall" - "testing" -) - -func TestGenerateMmapCommand(t *testing.T) { - commandCnt := getEnvOrDefault("COMMAND_CNT", "10") - commandCntNum, err := strconv.Atoi(commandCnt) - if err != nil { - t.Fatalf("parse COMMAND_CNT failed: %v", err) - return - } - filename := getEnvOrDefault("FILE_NAME", "/tmp/loongcollector/ebpfFileSecurityHook3.log") - f, err := os.Create(filename) - if err != nil { - panic(err) - } - fd := int(f.Fd()) - for i := 0; i < commandCntNum; i++ { - b, innerErr := syscall.Mmap(fd, 0, 20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) - if innerErr != nil { - panic(innerErr) - } - innerErr = syscall.Munmap(b) - if innerErr != nil { - panic(innerErr) - } - } - err = os.Remove(filename) - if err != nil { - t.Fatalf("remove file failed: %v", err) - return - } -} diff --git a/test/engine/trigger/generator/helper.go b/test/engine/trigger/generator/helper.go deleted file mode 100644 index 2514bb2614..0000000000 --- a/test/engine/trigger/generator/helper.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "crypto/rand" - "fmt" - "math/big" - "os" - "strconv" - "text/template" - - "github.com/pkg/errors" -) - -var Levels = []string{"ERROR", "INFO", "DEBUG", "WARNING"} - -type GenerateFileLogConfig struct { - GeneratedLogDir string - TotalLog int - Interval int - FileName string - Custom map[string]string -} - -func getGenerateFileLogConfigFromEnv(customKeys ...string) (*GenerateFileLogConfig, error) { - gneratedLogDir := getEnvOrDefault("GENERATED_LOG_DIR", "/tmp/loongcollector") - totalLog, err := strconv.Atoi(getEnvOrDefault("TOTAL_LOG", "100")) - if err != nil { - return nil, errors.Wrap(err, "parse TOTAL_LOG failed") - } - interval, err := strconv.Atoi(getEnvOrDefault("INTERVAL", "1")) - if err != nil { - return nil, errors.Wrap(err, "parse INTERVAL failed") - } - fileName := getEnvOrDefault("FILENAME", "default.log") - custom := make(map[string]string) - for _, key := range customKeys { - custom[key] = getEnvOrDefault(key, "") - } - return &GenerateFileLogConfig{ - GeneratedLogDir: gneratedLogDir, - TotalLog: totalLog, - Interval: interval, - FileName: fileName, - Custom: custom, - }, nil -} - -func string2Template(strings []string) []*template.Template { - templates := make([]*template.Template, len(strings)) - for i, str := range strings { - templates[i], _ = template.New(fmt.Sprintf("template_%d", i)).Parse(str) - } - return templates -} - -func getRandomLogLevel() string { - randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(Levels)))) - return Levels[randInt.Int64()] -} - -func getRandomMark() string { - marks := []string{"-", "F"} - randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(marks)))) - return marks[randInt.Int64()] -} - -func getEnvOrDefault(env, fallback string) string { - if value, ok := os.LookupEnv(env); ok { - return value - } - return fallback -} diff --git a/test/engine/trigger/generator/json_test.go b/test/engine/trigger/generator/json_test.go deleted file mode 100644 index 0b4ef51b33..0000000000 --- a/test/engine/trigger/generator/json_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "fmt" - "math/rand" - "os" - "strconv" - "testing" - "time" -) - -// TestGenerateJSONSingle will be executed in the environment being collected. -func TestGenerateJSONSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"POST","userAgent":"mozilla firefox","size":263} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"GET","userAgent":"go-sdk","size":569} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"HEAD","userAgent":"go-sdk","size":210} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"192.168.0.3","method":"PUT","userAgent":"curl/7.10","size":267} -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - var currentTime string - if i%2 == 0 { - currentTime = time.Now().Format("2006-01-02T15:04:05.999999999") - } else { - currentTime = strconv.FormatInt(time.Now().UnixNano()/1000, 10) - } - testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Time": currentTime, - }) - - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -func TestGenerateJSONMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get generate file log config from env failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"POST","userAgent":"mozilla firefox", -"size":263} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"GET","userAgent":"go-sdk", -"size":569} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"0.0.0.0","method":"HEAD","userAgent":"go-sdk", -"size":210} -`, - `{"mark":"{{.Mark}}","file":"file{{.FileNo}}","logNo":{{.LogNo}},"time":"{{.Time}}","ip":"192.168.0.3","method":"PUT","userAgent":"curl/7.10", -"size":267} -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - for i := 0; i < config.TotalLog; i++ { - currentTime := time.Now().Format("2006-01-02T15:04:05.999999999") - testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Time": currentTime, - }) - - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/generator/regex_test.go b/test/engine/trigger/generator/regex_test.go deleted file mode 100644 index c50e65dc6e..0000000000 --- a/test/engine/trigger/generator/regex_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "os" - "testing" - "time" - - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" -) - -// TestGenerateRegexLogSingle will be executed in the environment being collected. -func TestGenerateRegexLogSingle(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "HEAD / HTTP/2.0" 302 809 "未知" "这是一条消息,password:123456" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "GET /index.html HTTP/2.0" 200 139 "Mozilla/5.0" "这是一条消息,password:123456,这是第二条消息,password:00000" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 10.45.26.0 - [{{.Time}}] "PUT /index.html HTTP/1.1" 200 913 "curl/7.10" "这是一条消息" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 192.168.0.3 - [{{.Time}}] "PUT /dir/resource.txt HTTP/2.0" 501 355 "go-sdk" "这是一条消息,password:123456" -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -// TestGenerateRegexLogSingleGBK will be executed in the environment being collected. -func TestGenerateRegexLogSingleGBK(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - encoder := simplifiedchinese.GBK.NewEncoder() - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "HEAD / HTTP/2.0" 302 809 "未知" "这是一条消息,password:123456" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 127.0.0.1 - [{{.Time}}] "GET /index.html HTTP/2.0" 200 139 "Mozilla/5.0" "这是一条消息,password:123456,这是第二条消息,password:00000" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 10.45.26.0 - [{{.Time}}] "PUT /index.html HTTP/1.1" 200 913 "curl/7.10" "这是一条消息" -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} 192.168.0.3 - [{{.Time}}] "PUT /dir/resource.txt HTTP/2.0" 501 355 "go-sdk" "这是一条消息,password:123456" -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - var buffer bytes.Buffer - _ = testLogContentTmpl[logIndex].Execute(&buffer, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Mark": getRandomMark(), - "FileNo": fileNo, - "LogNo": logNo + i, - }) - data, err1 := io.ReadAll(transform.NewReader(&buffer, encoder)) - if err1 != nil { - t.Fatalf("encode log failed: %v", err1) - } - _, err := io.WriteString(file, string(data)) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} - -func TestGenerateRegexLogMultiline(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv() - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: exception happened -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:73) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: 发生异常 -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f2(RegexMultiLog.java:80) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:75) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - `{{.Mark}} file{{.FileNo}}:{{.LogNo}} [{{.Time}}] [{{.Level}}] java.lang.Exception: exception happened -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f5(RegexMultiLog.java:100) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f4(RegexMultiLog.java:96) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f3(RegexMultiLog.java:89) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f2(RegexMultiLog.java:82) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.f1(RegexMultiLog.java:75) -at com.aliyun.sls.devops.logGenerator.type.RegexMultiLog.run(RegexMultiLog.java:34) -at java.base/java.lang.Thread.run(Thread.java:833) -`, - }) - file, err := os.OpenFile(fmt.Sprintf("%s/%s", config.GeneratedLogDir, config.FileName), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - t.Fatalf("open file failed: %v", err) - return - } - defer file.Close() - logIndex := 0 - logNo := rand.Intn(10000) - fileNo := rand.Intn(10000) - location, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - t.Fatalf("load location failed: %v", err) - return - } - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(file, map[string]interface{}{ - "Time": time.Now().In(location).Format("2006-01-02T15:04:05.000000"), - "Level": getRandomLogLevel(), - "FileNo": fileNo, - "LogNo": logNo + i, - "Mark": getRandomMark(), - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/helper.go b/test/engine/trigger/helper.go index 8e0fb12171..696c63d0b2 100644 --- a/test/engine/trigger/helper.go +++ b/test/engine/trigger/helper.go @@ -14,11 +14,26 @@ package trigger import ( + "context" "fmt" + "path/filepath" + "strings" + "time" + + "github.com/alibaba/ilogtail/test/config" ) -const commandTemplate = "/usr/local/go/bin/go test -count=1 -v -run ^%s$ github.com/alibaba/ilogtail/test/engine/trigger/generator" +const commandTemplate = "python3 %s.py %s" + +func BeginTrigger(ctx context.Context) (context.Context, error) { + startTime := time.Now().Unix() + return context.WithValue(ctx, config.StartTimeContextKey, int32(startTime)), nil +} -func getRunTriggerCommand(triggerName string) string { - return fmt.Sprintf(commandTemplate, triggerName) +func GetRunTriggerCommand(scenrio, triggerName string, kvs ...string) string { + args := make([]string, 0) + for i := 0; i < len(kvs); i += 2 { + args = append(args, fmt.Sprintf("--%s", kvs[i]), kvs[i+1]) + } + return fmt.Sprintf(commandTemplate, filepath.Join(config.TestConfig.WorkDir, "engine", "trigger", scenrio, "remote_"+triggerName), strings.Join(args, " ")) } diff --git a/test/engine/trigger/log/file.go b/test/engine/trigger/log/file.go new file mode 100644 index 0000000000..ddea428021 --- /dev/null +++ b/test/engine/trigger/log/file.go @@ -0,0 +1,105 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package log + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/trigger" +) + +func RegexSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regex", path, totalLog, interval) +} + +func RegexSingleGBK(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regexGBK", path, totalLog, interval) +} + +func RegexMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "regexMultiline", path, totalLog, interval) +} + +func JSONSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "json", path, totalLog, interval) +} + +func JSONMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "jsonMultiline", path, totalLog, interval) +} + +func Apsara(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, "apsara", path, totalLog, interval) +} + +func DelimiterSingle(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { + return generate(ctx, "delimiter", path, totalLog, interval, "delimiter", delimiter, "quote", quote) +} + +func DelimiterMultiline(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { + return generate(ctx, "delimiterMultiline", path, totalLog, interval, "delimiter", delimiter, "quote", quote) +} + +func Nginx(ctx context.Context, rate, duration int, path string) (context.Context, error) { + return generateBenchmark(ctx, "nginx", path, rate, duration) +} + +func generate(ctx context.Context, mode, path string, count, interval int, customKV ...string) (context.Context, error) { + time.Sleep(3 * time.Second) + customKVString := make(map[string]string) + for i := 0; i < len(customKV); i += 2 { + customKVString[customKV[i]] = customKV[i+1] + } + jsonStr, err := json.Marshal(customKVString) + if err != nil { + return ctx, err + } + command := trigger.GetRunTriggerCommand("log", "file", "mode", mode, "path", path, "count", strconv.Itoa(count), "interval", strconv.Itoa(interval), "custom", wrapperCustomArgs(string(jsonStr))) + fmt.Println(command) + go func() { + if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { + fmt.Println(err) + } + }() + return ctx, nil +} + +func generateBenchmark(ctx context.Context, mode, path string, rate, duration int, customKV ...string) (context.Context, error) { + time.Sleep(3 * time.Second) + customKVString := make(map[string]string) + for i := 0; i < len(customKV); i += 2 { + customKVString[customKV[i]] = customKV[i+1] + } + jsonStr, err := json.Marshal(customKVString) + if err != nil { + return ctx, err + } + command := trigger.GetRunTriggerCommand("log", "file_benchmark", "mode", mode, "path", path, "rate", strconv.Itoa(rate), "duration", strconv.Itoa(duration), "custom", wrapperCustomArgs(string(jsonStr))) + if _, err := setup.Env.ExecOnSource(ctx, command); err != nil { + return ctx, err + } + return ctx, nil +} + +func wrapperCustomArgs(customArgs string) string { + fmt.Println(customArgs) + customArgs = strings.ReplaceAll(customArgs, "\\", "\\\\") + return "\"" + strings.ReplaceAll(customArgs, "\"", "\\\"") + "\"" +} diff --git a/test/engine/trigger/http.go b/test/engine/trigger/log/http_server.go similarity index 99% rename from test/engine/trigger/http.go rename to test/engine/trigger/log/http_server.go index 4a580e4c16..d2eec5b961 100644 --- a/test/engine/trigger/http.go +++ b/test/engine/trigger/log/http_server.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package trigger +package log import ( "context" diff --git a/test/engine/trigger/log/remote_file.py b/test/engine/trigger/log/remote_file.py new file mode 100644 index 0000000000..1415c48e36 --- /dev/null +++ b/test/engine/trigger/log/remote_file.py @@ -0,0 +1,172 @@ +import argparse +import json as jsonlib +import logging +import random +import time + +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime +from faker import Faker +from faker.providers import internet, user_agent, lorem, misc + + +def apsara(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}]\t[{get_random_level()}]\t[{random.randint(1, 10000)}]\t[/build/core/application/Application:{i}]\tfile:file{fileNo}\tlogNo:{i}\tmark:{get_random_mark()}\tmsg:{faker.sentence()}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def delimiter(args, logger, faker): + custom_args = args.custom + quote = custom_args.get('quote', '') + delimiter = custom_args.get('delimiter', ' ') + fileNo = random.randint(1, 1000) + for i in range(args.count): + logParts = [ + f'{quote}{get_random_mark()}{quote}', + f'{quote}file{fileNo}{quote}', + f'{quote}logNo:{i}{quote}', + f'{quote}{faker.ipv4()}{quote}', + f'{quote}{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}{quote}', + f'{quote}{faker.http_method()}{quote}', + f'{quote}{faker.uri_path()}{quote}', + f'{quote}HTTP/2.0{quote}', + f'{quote}{faker.http_status_code()}{quote}', + f'{quote}{random.randint(1, 10000)}{quote}', + f'{quote}{faker.user_agent()}{quote}' + ] + log = delimiter.join(logParts) + logger.info(log) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def delimiterMultiline(args, logger, faker): + custom_args = args.custom + quote = custom_args.get('quote', '') + delimiter = custom_args.get('delimiter', ' ') + fileNo = random.randint(1, 1000) + for i in range(args.count): + logParts = [ + f'{quote}{get_random_mark()}{quote}', + f'{quote}fi\nle{fileNo}{quote}', + f'{quote}logNo\n:{i}{quote}', + f'{quote}{faker.ipv4()}{quote}', + f'{quote}{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}{quote}', + f'{quote}{faker.http_method()}{quote}', + f'{quote}{faker.uri_path()}{quote}', + f'{quote}HT\nTP/2.0{quote}', + f'{quote}{faker.http_status_code()}{quote}', + f'{quote}{random.randint(1, 10000)}{quote}', + f'{quote}{faker.user_agent()}{quote}' + ] + log = delimiter.join(logParts) + logger.info(log) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def json(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{{"mark":"{get_random_mark()}", "file":"file{fileNo}", "logNo":{i}, "time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}", "ip": "{faker.ipv4()}", "method": "{faker.http_method()}", "userAgent": "{faker.user_agent()}", "size": {random.randint(1, 10000)}}}') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def jsonMultiline(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logParts = [ + f'"mark":"{get_random_mark()}"', + f'"file":"file{fileNo}"', + f'"logNo":{i}', + f'"time":"{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}"', + f'"ip": "{faker.ipv4()}"', + f'"method": "{faker.http_method()}"', + f'"userAgent": "{faker.user_agent()}"', + f'"size": {random.randint(1, 10000)}' + ] + log = '{' + ',\n'.join(logParts) + '}' + logger.info(log) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regex(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + logger.info(f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.user_agent()}" "{faker.sentence()}"') + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regexGBK(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + log = f'{get_random_mark()} file{fileNo}:{i} {faker.ipv4()} - [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] "{faker.http_method()} {faker.uri_path()} HTTP/2.0" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.user_agent()}" "{faker.sentence()}"' + logger.info(str(log.encode('gbk'))) + if args.interval > 0: + time.sleep(args.interval / 1000) + +def regexMultiline(args, logger, faker): + fileNo = random.randint(1, 1000) + for i in range(args.count): + multilineLog = '\n'.join(faker.sentences(nb=random.randint(1, 5))) + logger.info(f'{get_random_mark()} file{fileNo}:{i} [{datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}] [{get_random_level()}] java.lang.Exception: {multilineLog}') + + if args.interval > 0: + time.sleep(args.interval / 1000) + +def get_random_level(): + return random.choice(['DEBUG', 'INFO', 'WARNING', 'ERROR']) + +def get_random_mark(): + return random.choice(['-', 'F']) + +def parse_custom_arg_to_dict(custom_arg): + return jsonlib.loads(custom_arg) + +def main(): + parser = argparse.ArgumentParser(description='Log Generator Arg Parser') + parser.add_argument('--mode', type=str, default='regex', help='Log Type') + parser.add_argument('--path', type=str, default='default.log', help='Log Path') + parser.add_argument('--count', type=int, default=100, help='Log Count') + parser.add_argument('--interval', type=int, default=1, help='Log Interval (ms), < 0 means no interval') + parser.add_argument('--custom', type=parse_custom_arg_to_dict, help='Custom Args, in the format of json') + + args = parser.parse_args() + + logger = logging.getLogger('log_generator') + logger.setLevel(logging.INFO) + # 快速轮转来模拟比较极端的情况 + handler = TimedRotatingFileHandler(args.path, when="s", interval=5, backupCount=3) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + handler.flush = lambda: handler.stream.flush() + logger.addHandler(handler) + + # 随机生成器 + faker = Faker() + faker.add_provider(internet) + faker.add_provider(user_agent) + faker.add_provider(lorem) + faker.add_provider(misc) + + # 生成数据 + if args.mode == 'apsara': + apsara(args, logger, faker) + elif args.mode == 'delimiter': + delimiter(args, logger, faker) + elif args.mode == 'delimiterMultiline': + delimiterMultiline(args, logger, faker) + elif args.mode == 'json': + json(args, logger, faker) + elif args.mode == 'jsonMultiline': + jsonMultiline(args, logger, faker) + elif args.mode == 'regex': + regex(args, logger, faker) + elif args.mode == 'regexGBK': + regexGBK(args, logger, faker) + elif args.mode == 'regexMultiline': + regexMultiline(args, logger, faker) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/log/remote_file_benchmark.py b/test/engine/trigger/log/remote_file_benchmark.py new file mode 100644 index 0000000000..b9f53e00da --- /dev/null +++ b/test/engine/trigger/log/remote_file_benchmark.py @@ -0,0 +1,70 @@ +import argparse +import json +import logging +import math +import random +import time + +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime +from faker import Faker +from faker.providers import internet, user_agent, lorem, misc + +BATCH_SIZE = 100 + +def nginx(args, logger, faker): + startTime = time.perf_counter() + exampleLog = '' + for _ in range(BATCH_SIZE): + exampleLog += f'{faker.ipv4()} - - [{datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}\n"' + randomLogCost = (time.perf_counter() - startTime) / BATCH_SIZE + writeTimePerSecond = math.floor(args.rate * 1024 * 1024 / (len(exampleLog.encode('utf-8')))) + sleepInterval = 1 / writeTimePerSecond - randomLogCost + + startTime = datetime.now() + while True: + now = datetime.now() + fakeLog = f'{faker.ipv4()} - - [{now.strftime("%d/%b/%Y:%H:%M:%S %z")}] "{faker.http_method()} {faker.url()} HTTP/1.1" {faker.http_status_code()} {random.randint(1, 10000)} "{faker.url()}" "{faker.user_agent()}"\n' * BATCH_SIZE + logger.info(fakeLog[:-1]) + if sleepInterval > 0: + start = time.perf_counter() + while (time.perf_counter() - start) < sleepInterval: + pass + if (now - startTime).seconds > args.duration * 60: + break + +def parse_custom_arg_to_dict(custom_arg): + return json.loads(custom_arg) + +def main(): + parser = argparse.ArgumentParser(description='Log Generator Arg Parser') + parser.add_argument('--mode', type=str, default='nginx', help='Log Type') + parser.add_argument('--path', type=str, default='default.log', help='Log Path') + parser.add_argument('--rate', type=int, default=10, help='Log Generate Rate (MB/s)') + parser.add_argument('--duration', type=int, default=60, help='Log Generate Duration (min)') + parser.add_argument('--custom', nargs='*', type=parse_custom_arg_to_dict, help='Custom Args, in the format of key=value') + + args = parser.parse_args() + + logger = logging.getLogger('log_generator') + logger.setLevel(logging.INFO) + # 快速轮转来模拟比较极端的情况 + handler = TimedRotatingFileHandler(args.path, when="s", interval=70, backupCount=3) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + # 随机生成器 + faker = Faker() + faker.add_provider(internet) + faker.add_provider(user_agent) + faker.add_provider(lorem) + faker.add_provider(misc) + + # 生成数据 + if args.mode == 'nginx': + nginx(args, logger, faker) + + +if __name__ == '__main__': + main() diff --git a/test/engine/trigger/trigger.go b/test/engine/trigger/trigger.go deleted file mode 100644 index 63653c5cb6..0000000000 --- a/test/engine/trigger/trigger.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package trigger - -import ( - "context" - "strings" - "text/template" - "time" - - "github.com/alibaba/ilogtail/test/config" - "github.com/alibaba/ilogtail/test/engine/setup" -) - -const triggerTemplate = "cd {{.WorkDir}} && TOTAL_LOG={{.TotalLog}} INTERVAL={{.Interval}} FILENAME={{.Filename}} GENERATED_LOG_DIR={{.GeneratedLogDir}} {{.Custom}} {{.Command}}" - -func RegexSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogSingle") -} - -func RegexSingleGBK(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogSingleGBK") -} - -func RegexMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogMultiline") -} - -func JSONSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJSONSingle") -} - -func JSONMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJSONMultiline") -} - -func Apsara(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateApsara") -} - -func DelimiterSingle(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateDelimiterSingle", "Delimiter", delimiter, "Quote", quote) -} - -func DelimiterMultiline(ctx context.Context, totalLog int, path string, interval int, delimiter, quote string) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateDelimiterMultiline", "Delimiter", delimiter, "Quote", quote) -} - -func generate(ctx context.Context, totalLog int, path string, interval int, commandName string, customKV ...string) (context.Context, error) { - time.Sleep(3 * time.Second) - command := getRunTriggerCommand(commandName) - var triggerCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerTemplate)) - splittedPath := strings.Split(path, "/") - dir := strings.Join(splittedPath[:len(splittedPath)-1], "/") - filename := splittedPath[len(splittedPath)-1] - customString := strings.Builder{} - for i := 0; i < len(customKV); i++ { - customString.WriteString(customKV[i]) - customString.WriteString("=") - customString.WriteString(customKV[i+1]) - customString.WriteString(" ") - i++ - } - if err := template.Execute(&triggerCommand, map[string]interface{}{ - "WorkDir": config.TestConfig.WorkDir, - "TotalLog": totalLog, - "Interval": interval, - "GeneratedLogDir": dir, - "Filename": filename, - "Custom": customString.String(), - "Command": command, - }); err != nil { - return ctx, err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerCommand.String()); err != nil { - return ctx, err - } - return ctx, nil -} - -func BeginTrigger(ctx context.Context) (context.Context, error) { - startTime := time.Now().Unix() - return context.WithValue(ctx, config.StartTimeContextKey, int32(startTime)), nil -} diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000000..ea45cd03b7 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1 @@ +Faker \ No newline at end of file