feat: e2e test opt

Makefile.core.mk

@@ -233,14 +233,30 @@ include tools/lint.mk
 .PHONY: gateway-conformance-test
 gateway-conformance-test:
 
+# higress-conformance-test-prepare prepares the environment for higress conformance tests.
+.PHONY: higress-conformance-test-prepare
+higress-conformance-test-prepare: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev
+
 # higress-conformance-test runs ingress api conformance tests.
 .PHONY: higress-conformance-test
 higress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev run-higress-e2e-test delete-cluster
 
+# higress-conformance-test-clean cleans the environment for higress conformance tests.
+.PHONY: higress-conformance-test-clean
+higress-conformance-test-clean: $(tools/kind) delete-cluster
+
+# higress-wasmplugin-test-prepare prepares the environment for higress wasmplugin tests.
+.PHONY: higress-wasmplugin-test-prepare
+higress-wasmplugin-test-prepare: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin
+
 # higress-wasmplugin-test runs ingress wasmplugin tests.
 .PHONY: higress-wasmplugin-test
 higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-higress-e2e-test-wasmplugin delete-cluster
 
+# higress-wasmplugin-test-clean cleans the environment for higress wasmplugin tests.
+.PHONY: higress-wasmplugin-test-clean
+higress-wasmplugin-test-clean: $(tools/kind) delete-cluster
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)

test/e2e/conformance/tests/go-wasm-sni-misdirect.go

@@ -40,7 +40,7 @@ var WasmPluginsSniMisdirect = suite.ConformanceTest{
 		cliCertOut, cliKeyOut := cert.MustGenerateCertWithCA(t, cert.ClientCertType, caCert, caKey, nil)
 		fooSecret := kubernetes.ConstructTLSSecret("higress-conformance-infra", "foo-secret", svcCertOut.Bytes(), svcKeyOut.Bytes())
 		fooSecretCACert := kubernetes.ConstructCASecret("higress-conformance-infra", "foo-secret-cacert", caCertOut.Bytes())
-		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret, fooSecretCACert}, suite.Cleanup)
+		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret, fooSecretCACert}, true)
 
 		testcases := []http.Assertion{
 			{

test/e2e/conformance/tests/httproute-downstream-encryption.go

@@ -42,7 +42,7 @@ var HTTPRouteDownstreamEncryption = suite.ConformanceTest{
 		cliCertOut, cliKeyOut := cert.MustGenerateCertWithCA(t, cert.ClientCertType, caCert, caKey, nil)
 		fooSecret := kubernetes.ConstructTLSSecret("higress-conformance-infra", "foo-secret", svcCertOut.Bytes(), svcKeyOut.Bytes())
 		fooSecretCACert := kubernetes.ConstructCASecret("higress-conformance-infra", "foo-secret-cacert", caCertOut.Bytes())
-		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret, fooSecretCACert}, suite.Cleanup)
+		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret, fooSecretCACert}, true)
 
 		testcases := []http.Assertion{
 			{

test/e2e/conformance/tests/httproute-https-without-sni.go

@@ -39,7 +39,7 @@ var HTTPHttpsWithoutSni = suite.ConformanceTest{
 		_, _, caCert, caKey := cert.MustGenerateCaCert(t)
 		svcCertOut, svcKeyOut := cert.MustGenerateCertWithCA(t, cert.ServerCertType, caCert, caKey, []string{"foo.com"})
 		fooSecret := kubernetes.ConstructTLSSecret("higress-conformance-infra", "foo-secret", svcCertOut.Bytes(), svcKeyOut.Bytes())
-		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret}, suite.Cleanup)
+		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret}, true)
 
 		testcases := []http.Assertion{
 			{

test/e2e/conformance/utils/flags/flags.go

@@ -18,13 +18,12 @@ import (
 )
 
 var (
-	IngressClassName     = flag.String("ingress-class", "higress", "Name of IngressClass to use for tests")
-	ShowDebug            = flag.Bool("debug", false, "Whether to print debug logs")
-	CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
-	SupportedFeatures    = flag.String("supported-features", "", "Supported features included in conformance tests suites")
-	ExemptFeatures       = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
-	IsWasmPluginTest     = flag.Bool("isWasmPluginTest", false, "Determine if run wasm plugin conformance test")
-	WasmPluginType       = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP")
-	WasmPluginName       = flag.String("wasmPluginName", "", "Define wasm plugin name")
-	IsEnvoyConfigTest    = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test")
+	IngressClassName  = flag.String("ingress-class", "higress", "Name of IngressClass to use for tests")
+	ShowDebug         = flag.Bool("debug", false, "Whether to print debug logs")
+	SupportedFeatures = flag.String("supported-features", "", "Supported features included in conformance tests suites")
+	ExemptFeatures    = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
+	IsWasmPluginTest  = flag.Bool("isWasmPluginTest", false, "Determine if run wasm plugin conformance test")
+	WasmPluginType    = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP")
+	WasmPluginName    = flag.String("wasmPluginName", "", "Define wasm plugin name")
+	IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test")
 )

test/e2e/conformance/utils/kubernetes/apply.go

@@ -206,7 +206,7 @@ func (a Applier) MustDelete(t *testing.T, c client.Client, timeoutConfig config.
 		// 	}
 		// }
 
-		t.Logf("🏗 Deleting %s %s %s", uObj.GetName(), uObj.GetKind(), uObj.GetNamespace())
+		t.Logf("🚮 Deleting %s %s %s", uObj.GetName(), uObj.GetKind(), uObj.GetNamespace())
 		err = c.Delete(ctx, uObj)
 		require.NoErrorf(t, err, "error delete resource")
 	}

test/e2e/conformance/utils/suite/suite.go

@@ -32,7 +32,6 @@ type ConformanceTestSuite struct {
 	GatewayAddress    string
 	IngressClassName  string
 	Debug             bool
-	Cleanup           bool
 	BaseManifests     []string
 	Applier           kubernetes.Applier
 	SkipTests         sets.Set
@@ -56,10 +55,7 @@ type Options struct {
 	// Options for wasm extended features
 	WASMOptions
 
-	// CleanupBaseResources indicates whether or not the base test
-	// resources such as Gateways should be cleaned up after the run.
-	CleanupBaseResources bool
-	TimeoutConfig        config.TimeoutConfig
+	TimeoutConfig config.TimeoutConfig
 
 	// IsEnvoyConfigTest indicates whether or not the test is for envoy config
 	IsEnvoyConfigTest bool
@@ -105,7 +101,6 @@ func New(s Options) *ConformanceTestSuite {
 		RoundTripper:      roundTripper,
 		IngressClassName:  s.IngressClassName,
 		Debug:             s.Debug,
-		Cleanup:           s.CleanupBaseResources,
 		BaseManifests:     s.BaseManifests,
 		SupportedFeatures: s.SupportedFeatures,
 		GatewayAddress:    s.GatewayAddress,
@@ -126,27 +121,24 @@ func New(s Options) *ConformanceTestSuite {
 		}
 	}
 
+	suite.Applier.IngressClass = suite.IngressClassName
+
 	return suite
 }
 
 // Setup ensures the base resources required for conformance tests are installed
 // in the cluster. It also ensures that all relevant resources are ready.
 func (suite *ConformanceTestSuite) Setup(t *testing.T) {
-	t.Logf("📦 Test Setup: Ensuring IngressClass has been accepted")
-
-	suite.Applier.IngressClass = suite.IngressClassName
-
 	t.Logf("📦 Test Setup: Applying base manifests")
-
 	for _, baseManifest := range suite.BaseManifests {
-		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, baseManifest, suite.Cleanup)
+		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, baseManifest, false)
 	}
 
 	t.Logf("📦 Test Setup: Applying programmatic resources")
 	secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"})
-	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
+	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, false)
 	secret = kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-infra", "tls-validity-checks-certificate", []string{"*"})
-	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
+	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, false)
 
 	t.Logf("📦 Test Setup: Ensuring Pods from base manifests are ready")
 	namespaces := []string{
@@ -159,7 +151,7 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
 	t.Logf("🌱 Supported Features: %+v", suite.SupportedFeatures.UnsortedList())
 }
 
-// RunWithTests runs the provided set of conformance tests.
+// Run runs the provided set of conformance tests.
 func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 	t.Logf("🚀 Start Running %d Test Cases: \n\n%s", len(tests), globalConformanceTestsListInfo(tests))
 	for _, test := range tests {
@@ -169,6 +161,12 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 	}
 }
 
+// Cleanup ensures that all resources created by the test suite are cleaned up.
+func (suite *ConformanceTestSuite) Cleanup(t *testing.T) {
+	t.Logf("🧹 Test Cleanup: Ensuring all resources are cleaned up")
+	suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, "base/manifests.yaml")
+}
+
 func globalConformanceTestsListInfo(tests []ConformanceTest) string {
 	var cases string
 	for index, test := range tests {

test/e2e/e2e_test.go

@@ -16,19 +16,19 @@ package test
 
 import (
 	"flag"
+	"github.com/alibaba/higress/test/e2e/conformance/tests"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/networking/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
-	"github.com/alibaba/higress/test/e2e/conformance/tests"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/flags"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/suite"
 )
 
-func TestHigressConformanceTests(t *testing.T) {
+func TestPrepareHigressConformanceTests(t *testing.T) {
 	flag.Parse()
 
 	cfg, err := config.GetConfig()
@@ -40,10 +40,9 @@ func TestHigressConformanceTests(t *testing.T) {
 	require.NoError(t, v1.AddToScheme(client.Scheme()))
 
 	cSuite := suite.New(suite.Options{
-		Client:               client,
-		IngressClassName:     *flags.IngressClassName,
-		Debug:                *flags.ShowDebug,
-		CleanupBaseResources: *flags.CleanupBaseResources,
+		Client:           client,
+		IngressClassName: *flags.IngressClassName,
+		Debug:            *flags.ShowDebug,
 		WASMOptions: suite.WASMOptions{
 			IsWasmPluginTest: *flags.IsWasmPluginTest,
 			WasmPluginName:   *flags.WasmPluginName,
@@ -55,5 +54,60 @@ func TestHigressConformanceTests(t *testing.T) {
 	})
 
 	cSuite.Setup(t)
+}
+
+func TestRunHigressConformanceTests(t *testing.T) {
+	flag.Parse()
+
+	cfg, err := config.GetConfig()
+	require.NoError(t, err)
+
+	client, err := client.New(cfg, client.Options{})
+	require.NoError(t, err)
+
+	require.NoError(t, v1.AddToScheme(client.Scheme()))
+
+	cSuite := suite.New(suite.Options{
+		Client:           client,
+		IngressClassName: *flags.IngressClassName,
+		Debug:            *flags.ShowDebug,
+		WASMOptions: suite.WASMOptions{
+			IsWasmPluginTest: *flags.IsWasmPluginTest,
+			WasmPluginName:   *flags.WasmPluginName,
+			WasmPluginType:   *flags.WasmPluginType,
+		},
+		GatewayAddress:             "localhost",
+		EnableAllSupportedFeatures: true,
+		IsEnvoyConfigTest:          *flags.IsEnvoyConfigTest,
+	})
+
 	cSuite.Run(t, tests.ConformanceTests)
 }
+
+func TestCleanHigressConformanceTests(t *testing.T) {
+	flag.Parse()
+
+	cfg, err := config.GetConfig()
+	require.NoError(t, err)
+
+	client, err := client.New(cfg, client.Options{})
+	require.NoError(t, err)
+
+	require.NoError(t, v1.AddToScheme(client.Scheme()))
+
+	cSuite := suite.New(suite.Options{
+		Client:           client,
+		IngressClassName: *flags.IngressClassName,
+		Debug:            *flags.ShowDebug,
+		WASMOptions: suite.WASMOptions{
+			IsWasmPluginTest: *flags.IsWasmPluginTest,
+			WasmPluginName:   *flags.WasmPluginName,
+			WasmPluginType:   *flags.WasmPluginType,
+		},
+		GatewayAddress:             "localhost",
+		EnableAllSupportedFeatures: true,
+		IsEnvoyConfigTest:          *flags.IsEnvoyConfigTest,
+	})
+
+	cSuite.Cleanup(t)
+}