feat: e2e test opt (#746)

Signed-off-by: sjcsjc123 <[email protected]>

.licenserc.yaml

@@ -26,6 +26,7 @@ header:
     - 'VERSION'
     - 'tools/'
     - 'test/README.md'
+    - 'test/README_CN.md'
     - 'cmd/hgctl/config/testdata/config'
     - 'pkg/cmd/hgctl/manifests'
 

Makefile.core.mk

@@ -233,14 +233,30 @@ include tools/lint.mk
 .PHONY: gateway-conformance-test
 gateway-conformance-test:
 
+# higress-conformance-test-prepare prepares the environment for higress conformance tests.
+.PHONY: higress-conformance-test-prepare
+higress-conformance-test-prepare: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev
+
 # higress-conformance-test runs ingress api conformance tests.
 .PHONY: higress-conformance-test
 higress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev run-higress-e2e-test delete-cluster
 
+# higress-conformance-test-clean cleans the environment for higress conformance tests.
+.PHONY: higress-conformance-test-clean
+higress-conformance-test-clean: $(tools/kind) delete-cluster
+
+# higress-wasmplugin-test-prepare prepares the environment for higress wasmplugin tests.
+.PHONY: higress-wasmplugin-test-prepare
+higress-wasmplugin-test-prepare: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin
+
 # higress-wasmplugin-test runs ingress wasmplugin tests.
 .PHONY: higress-wasmplugin-test
 higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-higress-e2e-test-wasmplugin delete-cluster
 
+# higress-wasmplugin-test-clean cleans the environment for higress wasmplugin tests.
+.PHONY: higress-wasmplugin-test-clean
+higress-wasmplugin-test-clean: $(tools/kind) delete-cluster
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)
@@ -270,6 +286,17 @@ kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster us
 	tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2
 	tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
 	tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest
+
+# run-higress-e2e-test-setup starts to setup ingress e2e tests.
+.PHONT: run-higress-e2e-test-setup
+run-higress-e2e-test-setup:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --test-area=setup
+
 # run-higress-e2e-test starts to run ingress e2e tests.
 .PHONY: run-higress-e2e-test
 run-higress-e2e-test:
@@ -278,14 +305,64 @@ run-higress-e2e-test:
 	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
-	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --test-area=all
 
-# run-higress-e2e-test starts to run ingress e2e tests.
+# run-higress-e2e-test-run starts to run ingress e2e conformance tests.
+.PHONY: run-higress-e2e-test-run
+run-higress-e2e-test-run:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --test-area=run
+
+# run-higress-e2e-test-clean starts to clean ingress e2e tests.
+.PHONY: run-higress-e2e-test-clean
+run-higress-e2e-test-clean:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --test-area=clean
+
+# run-higress-e2e-test-wasmplugin-setup starts to prepare ingress e2e tests.
+.PHONY: run-higress-e2e-test-wasmplugin-setup
+run-higress-e2e-test-wasmplugin-setup:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=setup
+
+# run-higress-e2e-test-wasmplugin starts to run ingress e2e tests.
 .PHONY: run-higress-e2e-test-wasmplugin
 run-higress-e2e-test-wasmplugin:
 	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
 	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
-	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true
+	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=all
+
+# run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests.
+.PHONY: run-higress-e2e-test-wasmplugin-run
+run-higress-e2e-test-wasmplugin-run:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=run
+
+# run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests.
+.PHONY: run-higress-e2e-test-wasmplugin-clean
+run-higress-e2e-test-wasmplugin-clean:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean

test/README.md

@@ -43,6 +43,26 @@ And after that, you should add your defined HigressConformanceTest to `e2e_test.
 
 You can understand it quickly just by looking at codes in `test/ingress/conformance/tests/httproute-simple-same-namespace.go` and `test/ingress/conformance/tests/httproute-simple-same-namespace.yaml`, and try to write one.
 
+### How to Implement Test Environment Reusability
+
+The test environment reusability is primarily achieved through the following targets in the Makefile:
+
+1. **make higress-conformance-test:** Used to run the entire Conformance testing process, including setting up the test environment, executing test cases, and cleaning up the test environment.
+   - **make higress-conformance-test-prepare:** Can be used to set up the environment for deployments such as higress-controller, higress-gateway, etc.
+   - **make run-higress-e2e-test:** Used to run the test cases.
+      - **make run-higress-e2e-test-setup:** Can be used to install the basic resources required for the test cases, such as nacos, dubbo, etc.
+      - **make run-higress-e2e-test-run:** Used to execute the test cases.
+      - **make run-higress-e2e-test-clean:** Can be used to clean up the basic resources installed during the setup phase of the test cases.
+   - **make higress-conformance-test-clean:** Used to clean up the test environment for deployments like higress-controller, higress-gateway, etc.
+
+2. **make higress-wasmplugin-test:** Used to run the entire WasmPlugin testing process, including setting up the test environment, compiling WasmPlugin plugins, executing test cases, and cleaning up the test environment.
+   - **make higress-wasmplugin-test-prepare:** Can be used to set up the environment for deployments such as higress-controller, higress-gateway, and compile WasmPlugin plugins.
+   - **make run-higress-e2e-test-wasmplugin:** Used to run the test cases.
+      - **make run-higress-e2e-test-wasmplugin-setup:** Can be used to install the basic resources required for the test cases, such as nacos, dubbo, etc.
+      - **make run-higress-e2e-test-wasmplugin-run:** Used to execute the test cases.
+      - **make run-higress-e2e-test-wasmplugin-clean:** Can be used to clean up the basic resources installed during the setup phase of the test cases.
+   - **make higress-wasmplugin-test-clean:** Used to clean up the test environment for deployments like higress-controller, higress-gateway, etc.
+
 ## Gateway APIs Conformance Tests
 
 Gateway API Conformance tests are based on the suite provided by `kubernetes-sig/gateway-api`, we can reuse that,

test/README_CN.md

@@ -0,0 +1,76 @@
+# Higress E2E 测试
+
+Higress E2E 测试主要关注两个部分：
+
++ Ingress API 的一致性测试
++ Gateway API 的一致性测试
+
+## Ingress API 一致性测试
+
+### 架构
+
+![ingress-arch](./e2e/arch.png)
+
+### 工作流程
+
+![ingress-workflow](./e2e/pipeline.png)
+
+Higress 提供了运行 Ingress API 一致性测试和 wasmplugin 测试的 make 目标，
+
++ API 测试: `make higress-conformance-test`
++ WasmPlugin 测试: `make higress-wasmplugin-test`
+    + 为测试构建所有 GO WasmPlugins: `make higress-wasmplugin-test`
+    + 仅为一个 GO WasmPlugin 构建测试: `PLUGIN_NAME=request-block make higress-wasmplugin-test`
+    + 仅为一个 CPP WasmPlugin 构建测试: `PLUGIN_TYPE=CPP PLUGIN_NAME=key_auth make higress-wasmplugin-test`
+
+可以分为以下步骤：
+
+1. delete-cluster: 检查是否有未删除的 kind 集群。
+2. create-cluster: 创建一个新的 kind 集群。
+3. docker-build: 构建 higress 的开发镜像。
+4. kube-load-image: 将 dev higress-controller 镜像加载到 kind 集群中。
+5. install-dev: 使用 helm 安装带有 dev 镜像的 higress-controller，并安装最新的 higress-gateway、istiod。
+6. run-e2e-test:
+    1. 设置一致性测试套件，例如在 `e2e_test.go` / `higressTests Slice` 中定义我们想要运行的一致性测试。我们选择打开的每个测试都在 `test/ingress/conformance/tests` 中定义。
+    2. 准备资源并将它们安装到集群中，例如后端服务/部署。
+    3. 在 `e2e_test.go` / `higressTests Slice` 中加载我们选择打开的一致性测试，并逐个运行它们，如果不符合预期，则失败。
+
+### 如何编写测试用例
+
+要添加新的测试用例，首先需要在 `test/ingress/conformance/tests` 中添加 `xxx.go` 和 `xxx.yaml`。`xxx.yaml` 是您需要在集群中应用的 Ingress 资源，`xxx.go` 定义了 HigressConformanceTest。
+
+然后，您应该将您定义的 HigressConformanceTest 添加到 `e2e_test.go` / `higressTests Slice` 中。
+
+通过查看 `test/ingress/conformance/tests/httproute-simple-same-namespace.go` 和 `test/ingress/conformance/tests/httproute-simple-same-namespace.yaml` 中的代码，您可以快速了解并尝试编写一个测试用例。
+
+### 如何实现测试环境的复用
+
+主要通过 Makefile 中的以下几个目标实现：
+
+1. **make higress-conformance-test:** 用于运行整个 Conformance 测试流程，包括搭建测试环境、运行测试用例、清理测试环境。
+    - **make higress-conformance-test-prepare:** 可用于搭建 higress-controller、higress-gateway 等 deployment 的环境。
+    - **make run-higress-e2e-test:** 可用于运行测试用例。
+        - **make run-higress-e2e-test-setup:** 可用于安装测试用例所需的基础资源，例如 nacos、dubbo 等。
+        - **make run-higress-e2e-test-run:** 可用于运行测试用例。
+        - **make run-higress-e2e-test-clean:** 可用于清理测试用例在 setup 阶段所安装的基础资源。
+    - **make higress-conformance-test-clean:** 可用于清理 higress-controller、higress-gateway 等 deployment 的测试环境。
+
+2. **make higress-wasmplugin-test:** 用于运行整个 WasmPlugin 测试流程，包括搭建测试环境、编译 WasmPlugin 插件、运行测试用例、清理测试环境。
+    - **make higress-wasmplugin-test-prepare:** 可用于搭建 higress-controller、higress-gateway 等 deployment 的环境，并编译 WasmPlugin 插件。
+    - **make run-higress-e2e-test-wasmplugin:** 可用于运行测试用例。
+        - **make run-higress-e2e-test-wasmplugin-setup:** 可用于安装测试用例所需的基础资源，例如 nacos、dubbo 等。
+        - **make run-higress-e2e-test-wasmplugin-run:** 可用于运行测试用例。
+        - **make run-higress-e2e-test-wasmplugin-clean:** 可用于清理测试用例在 setup 阶段所安装的基础资源。
+    - **make higress-wasmplugin-test-clean:** 可用于清理 higress-controller、higress-gateway 等 deployment 的测试环境。
+
+## Gateway API 一致性测试
+
+Gateway API 一致性测试基于 `kubernetes-sig/gateway-api` 提供的套件，我们可以重复使用它，并决定我们需要打开哪些 Gateway API 的一致性测试。
+
+此 API 包含一系列广泛的功能和用例，并已得到广泛实现。
+这个大的功能集和各种实现的结合需要明确的一致性定义和测试，以确保在任何地方使用该 API 时都提供一致的体验。
+
+Gateway API 包括一组一致性测试。这些测试创建具有指定 GatewayClass 的一系列 Gateways 和 Routes，并测试实现是否符合 API 规范。
+
+每个发布版本都包含一组一致性测试，随着 API 的演进，这些测试将不断扩展。
+目前，一致性测试覆盖了标准通道中的大多数核心功能，以及一些扩展功能。

test/e2e/conformance/utils/flags/flags.go

@@ -27,4 +27,5 @@ var (
 	WasmPluginType       = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP")
 	WasmPluginName       = flag.String("wasmPluginName", "", "Define wasm plugin name")
 	IsEnvoyConfigTest    = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test")
+	TestArea             = flag.String("test-area", "all", "Test area to run, like all to run setup/run/clean, setup to prepare test environment, run to run test cases, clean to clean test environment")
 )

test/e2e/conformance/utils/kubernetes/apply.go

@@ -206,7 +206,7 @@ func (a Applier) MustDelete(t *testing.T, c client.Client, timeoutConfig config.
 		// 	}
 		// }
 
-		t.Logf("🏗 Deleting %s %s %s", uObj.GetName(), uObj.GetKind(), uObj.GetNamespace())
+		t.Logf("🚮 Deleting %s %s %s", uObj.GetName(), uObj.GetKind(), uObj.GetNamespace())
 		err = c.Delete(ctx, uObj)
 		require.NoErrorf(t, err, "error delete resource")
 	}

test/e2e/conformance/utils/suite/suite.go

@@ -24,6 +24,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	TestAreaAll   = "all"
+	TestAreaSetup = "setup"
+	TestAreaRun   = "run"
+	TessAreaClean = "clean"
+)
+
 // ConformanceTestSuite defines the test suite used to run Gateway API
 // conformance tests.
 type ConformanceTestSuite struct {
@@ -159,7 +166,7 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
 	t.Logf("🌱 Supported Features: %+v", suite.SupportedFeatures.UnsortedList())
 }
 
-// RunWithTests runs the provided set of conformance tests.
+// Run runs the provided set of conformance tests.
 func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 	t.Logf("🚀 Start Running %d Test Cases: \n\n%s", len(tests), globalConformanceTestsListInfo(tests))
 	for _, test := range tests {
@@ -169,6 +176,17 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 	}
 }
 
+// Clean cleans up the base resources installed by Setup.
+func (suite *ConformanceTestSuite) Clean(t *testing.T) {
+	if suite.Cleanup {
+		t.Logf("🧹 Test Cleanup: Ensuring base resources have been cleaned up")
+
+		for _, baseManifest := range suite.BaseManifests {
+			suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, baseManifest)
+		}
+	}
+}
+
 func globalConformanceTestsListInfo(tests []ConformanceTest) string {
 	var cases string
 	for index, test := range tests {

test/e2e/e2e_test.go

@@ -16,6 +16,7 @@ package test
 
 import (
 	"flag"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -54,6 +55,19 @@ func TestHigressConformanceTests(t *testing.T) {
 		IsEnvoyConfigTest:          *flags.IsEnvoyConfigTest,
 	})
 
-	cSuite.Setup(t)
-	cSuite.Run(t, tests.ConformanceTests)
+	// Run the test suite
+	testArea := *flags.TestArea
+	testArea = strings.ToLower(testArea)
+	switch testArea {
+	case suite.TestAreaAll:
+		cSuite.Setup(t)
+		cSuite.Run(t, tests.ConformanceTests)
+	case suite.TestAreaRun:
+		cSuite.Run(t, tests.ConformanceTests)
+	case suite.TestAreaSetup:
+		cSuite.Cleanup = false
+		cSuite.Setup(t)
+	case suite.TessAreaClean:
+		cSuite.Clean(t)
+	}
 }