test: network chaos and restart agent

test/config/config.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/alibaba/ilogtail/pkg/logger"
@@ -46,7 +47,6 @@ type Config struct {
 	AccessKeySecret string        `mapstructure:"access_key_secret" yaml:"access_key_secret"`
 	Endpoint        string        `mapstructure:"endpoint" yaml:"endpoint"`
 	Aliuid          string        `mapstructure:"aliuid" yaml:"aliuid"`
-	QueryEndpoint   string        `mapstructure:"query_endpoint" yaml:"query_endpoint"`
 	Region          string        `mapstructure:"region" yaml:"region"`
 	RetryTimeout    time.Duration `mapstructure:"retry_timeout" yaml:"retry_timeout"`
 }
@@ -94,11 +94,18 @@ func ParseConfig() {
 	TestConfig.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET")
 	TestConfig.Endpoint = os.Getenv("ENDPOINT")
 	TestConfig.Aliuid = os.Getenv("ALIUID")
-	TestConfig.QueryEndpoint = os.Getenv("QUERY_ENDPOINT")
 	TestConfig.Region = os.Getenv("REGION")
 	timeout, err := strconv.ParseInt(os.Getenv("RETRY_TIMEOUT"), 10, 64)
 	if err != nil {
 		timeout = 60
 	}
 	TestConfig.RetryTimeout = time.Duration(timeout) * time.Second
 }
+
+func GetQueryEndpoint() string {
+	idx := strings.Index(TestConfig.Endpoint, "-intranet")
+	if idx == -1 {
+		return TestConfig.Endpoint
+	}
+	return TestConfig.Endpoint[:idx] + TestConfig.Endpoint[idx+9:]
+}

test/config/context.go

@@ -23,4 +23,5 @@ const (
 	CurrentWorkingDeploymentKey ContextKey = "currentWorkingDeployment"
 	QueryKey                    ContextKey = "query"
 	AgentPIDKey                 ContextKey = "agentPID"
+	EndpointIPKey               ContextKey = "endpointIP"
 )

test/engine/cleanup/chaos.go

@@ -0,0 +1,74 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package cleanup
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/alibaba/ilogtail/test/engine/setup"
+)
+
+type ChaosStatus struct {
+	Code    int                 `json:"code"`
+	Success bool                `json:"success"`
+	Result  []map[string]string `json:"result"`
+}
+
+func DestoryAllChaos(ctx context.Context) (context.Context, error) {
+	switch setup.Env.GetType() {
+	case "host":
+		command := "/opt/chaosblade/blade status --type create --status Success"
+		response, err := setup.Env.ExecOnLogtail(command)
+		if err != nil {
+			return ctx, err
+		}
+		var status ChaosStatus
+		if err = json.Unmarshal([]byte(response), &status); err != nil {
+			return ctx, err
+		}
+		for _, result := range status.Result {
+			command = "/opt/chaosblade/blade destroy " + result["Uid"]
+			if _, err := setup.Env.ExecOnLogtail(command); err != nil {
+				fmt.Println("Destroy chaos failed: ", err)
+			}
+		}
+	case "daemonset", "deployment":
+		k8sEnv := setup.Env.(*setup.K8sEnv)
+		chaosDir := filepath.Join("test_cases", "chaos")
+		err := filepath.Walk(chaosDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			if filepath.Ext(path) != ".yaml" {
+				return nil
+			}
+			return k8sEnv.Delete(path[len("test_cases/"):])
+		})
+		if err != nil {
+			return ctx, err
+		}
+		// delete chaosDir
+		if err = os.RemoveAll(chaosDir); err != nil {
+			return ctx, err
+		}
+	}
+	return ctx, nil
+}

test/engine/cleanup/helper.go

@@ -15,6 +15,7 @@ package cleanup
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -44,10 +45,20 @@ func All() {
 		return
 	}
 	ctx := context.TODO()
-	_, _ = control.RemoveAllLocalConfig(ctx)
-	_, _ = AllGeneratedLog(ctx)
-	_, _ = GoTestCache(ctx)
-	_, _ = DeleteContainers(ctx)
+	red := "\033[31m"
+	reset := "\033[0m"
+	if _, err := control.RemoveAllLocalConfig(ctx); err != nil {
+		fmt.Println(red + err.Error() + reset)
+	}
+	if _, err := AllGeneratedLog(ctx); err != nil {
+		fmt.Println(red + err.Error() + reset)
+	}
+	if _, err := DestoryAllChaos(ctx); err != nil {
+		fmt.Println(red + err.Error() + reset)
+	}
+	if _, err := DeleteContainers(ctx); err != nil {
+		fmt.Println(red + err.Error() + reset)
+	}
 	if subscriber.TestSubscriber != nil {
 		_ = subscriber.TestSubscriber.Stop()
 	}

test/engine/control/agent.go

@@ -0,0 +1,48 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package control
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/alibaba/ilogtail/test/config"
+	"github.com/alibaba/ilogtail/test/engine/setup"
+)
+
+func RestartAgent(ctx context.Context) (context.Context, error) {
+	if _, err := setup.Env.ExecOnLogtail("/etc/init.d/loongcollectord restart"); err != nil {
+		return ctx, err
+	}
+	return setup.SetAgentPID(ctx)
+}
+
+func ForceRestartAgent(ctx context.Context) (context.Context, error) {
+	currentPID := ctx.Value(config.AgentPIDKey)
+	if currentPID != nil {
+		currentPIDs := strings.Split(strings.TrimSpace(currentPID.(string)), "\n")
+		for _, pid := range currentPIDs {
+			if _, err := setup.Env.ExecOnLogtail("kill -9 " + pid); err != nil {
+				fmt.Println("Force kill agent pid failed: ", err)
+			}
+		}
+	} else {
+		fmt.Println("No agent pid found, skip force restart")
+	}
+	if _, err := setup.Env.ExecOnLogtail("/etc/init.d/loongcollectord restart"); err != nil {
+		return ctx, err
+	}
+	return setup.SetAgentPID(ctx)
+}

test/engine/setup/chaos/network.go

@@ -0,0 +1,156 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package chaos
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"text/template"
+
+	"github.com/alibaba/ilogtail/test/engine/setup"
+)
+
+const (
+	// networkDelayCRD
+	networkDelayCRDTmpl = `
+apiVersion: chaosblade.io/v1alpha1
+kind: ChaosBlade
+metadata:
+  name: delay-pod-network
+spec:
+  experiments:
+    - scope: pod
+      target: network
+      action: delay
+      desc: "delay pod network"
+      matchers:
+        - name: labels
+          value: ["{{.PodLabel}}"]
+        - name: namespace
+          value: ["kube-system"]
+        - name: interface
+          value: ["eth0"]
+        - name: destination-ip
+          value: ["{{.Percent}}"]
+        - name: time
+          value: ["{{.Time}}"]
+`
+
+	// networkLossCRD
+	networkLossCRDTmpl = `
+apiVersion: chaosblade.io/v1alpha1
+kind: ChaosBlade
+metadata:
+  name: loss-pod-network
+spec:
+  experiments:
+    - scope: pod
+      target: network
+      action: loss
+      desc: "loss pod network"
+      matchers:
+        - name: labels
+          value: ["{{.PodLabel}}"]
+        - name: namespace
+          value: ["kube-system"]
+        - name: interface
+          value: ["eth0"]
+        - name: percent
+          value: ["{{.Percent}}"]
+        - name: exclude-port
+          value: ["22"]
+        - name: destination-ip
+          value: ["{{.Ip}}"]
+`
+)
+
+func NetworkDelay(ctx context.Context, time int, ip string) (context.Context, error) {
+	switch setup.Env.GetType() {
+	case "host":
+		command := "/opt/chaosblade/blade create network delay --time " + strconv.FormatInt(int64(time), 10) + " --exclude-port 22 --interface eth0 --destination-ip " + ip
+		_, err := setup.Env.ExecOnLogtail(command)
+		if err != nil {
+			return ctx, err
+		}
+	case "daemonset", "deployment":
+		dir := filepath.Join("test_cases", "chaos")
+		filename := "loss-pod-network.yaml"
+		_ = os.Mkdir(dir, 0750)
+		file, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec
+		if err != nil {
+			return ctx, err
+		}
+		defer file.Close() //nolint:gosec
+
+		networkDelayCRD, _ := template.New("networkDelay").Parse(networkDelayCRDTmpl)
+		if err = networkDelayCRD.Execute(file, map[string]string{
+			"PodLabel": getLoongCollectorPodLabel(),
+			"Time":     strconv.FormatInt(int64(time), 10),
+			"Ip":       ip,
+		}); err != nil {
+			return ctx, err
+		}
+		k8sEnv := setup.Env.(*setup.K8sEnv)
+		if err := k8sEnv.Apply(filepath.Join("chaos", filename)); err != nil {
+			return ctx, err
+		}
+	}
+	return ctx, nil
+}
+
+func NetworkLoss(ctx context.Context, percentage int, ip string) (context.Context, error) {
+	switch setup.Env.GetType() {
+	case "host":
+		command := "/opt/chaosblade/blade create network loss --percent " + strconv.FormatInt(int64(percentage), 10) + " --exclude-port 22 --interface eth0 --destination-ip " + ip
+		_, err := setup.Env.ExecOnLogtail(command)
+		if err != nil {
+			return ctx, err
+		}
+	case "daemonset", "deployment":
+		dir := filepath.Join("test_cases", "chaos")
+		filename := "loss-pod-network.yaml"
+		_ = os.Mkdir(dir, 0750)
+		file, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec
+		if err != nil {
+			return ctx, err
+		}
+		defer file.Close() //nolint:gosec
+
+		networkLossCRD, _ := template.New("networkLoss").Parse(networkLossCRDTmpl)
+		if err = networkLossCRD.Execute(file, map[string]string{
+			"PodLabel": getLoongCollectorPodLabel(),
+			"Percent":  strconv.FormatInt(int64(percentage), 10),
+			"Ip":       ip,
+		}); err != nil {
+			return ctx, err
+		}
+		k8sEnv := setup.Env.(*setup.K8sEnv)
+		if err := k8sEnv.Apply(filepath.Join("chaos", filename)); err != nil {
+			return ctx, err
+		}
+	}
+	return ctx, nil
+}
+
+func getLoongCollectorPodLabel() string {
+	var PodLabel string
+	if setup.Env.GetType() == "daemonset" {
+		PodLabel = "k8s-app=logtail-ds"
+	} else if setup.Env.GetType() == "deployment" {
+		PodLabel = "k8s-app=loongcollector-cluster"
+	}
+	return PodLabel
+}