Add benchmark workflows

.github/workflows/benchmark.yaml

@@ -0,0 +1,54 @@
+name: Benchmark
+
+on:
+  pull_request:
+    paths:
+    - pkg/**/*
+    - cmd/**/*
+    - test/**/*
+    - hack/**/*
+    - kustomize/**/*
+    - go.mod
+    - .github/workflows/benchmark.yaml
+    - '!hack/releases-helm-chart.sh'
+  push:
+    paths:
+    - pkg/**/*
+    - cmd/**/*
+    - test/**/*
+    - hack/**/*
+    - kustomize/**/*
+    - go.mod
+    - .github/workflows/benchmark.yaml
+    - '!hack/releases-helm-chart.sh'
+
+env:
+  CGO_ENABLED: "0"
+  GO_VERSION: "1.23.0"
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GO_VERSION }}
+
+    - name: Test Benchmark
+      shell: bash
+      run: |
+        ./hack/e2e-test.sh e2e/kwokctl/benchmark
+
+    - name: Test Benchmark Hack
+      shell: bash
+      run: |
+        ./hack/e2e-test.sh e2e/kwokctl/benchmark-hack
+
+    - name: Upload logs
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: kwok-logs-benchmark
+        path: ${{ github.workspace }}/logs

.github/workflows/test.yaml

@@ -270,12 +270,6 @@ jobs:
         fi
         ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}
 
-    - name: Test Benchmark
-      if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }}
-      shell: bash
-      run: |
-        ./hack/e2e-test.sh e2e/kwokctl/benchmark
-
     - name: Test Auto Detect
       if: ${{ matrix.kwokctl-runtime == 'binary' }}
       shell: bash

test/e2e/benchmark.go

@@ -120,10 +120,10 @@ func scaleCreateNode(ctx context.Context, t *testing.T, kwokctlPath string, name
 func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder {
 	return features.New("Benchmark").
 		Assess("Create nodes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
-			ctx0, cancel := context.WithTimeout(ctx, 120*time.Second)
+			ctx0, cancel := context.WithTimeout(ctx, 180*time.Second)
 			defer cancel()
 
-			err := scaleCreateNode(ctx0, t, kwokctlPath, clusterName, 2000)
+			err := scaleCreateNode(ctx0, t, kwokctlPath, clusterName, 5000)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -133,7 +133,7 @@ func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder {
 			ctx0, cancel := context.WithTimeout(ctx, 240*time.Second)
 			defer cancel()
 
-			err := scaleCreatePod(ctx0, t, kwokctlPath, clusterName, 5000)
+			err := scaleCreatePod(ctx0, t, kwokctlPath, clusterName, 10000)
 			if err != nil {
 				t.Fatal(err)
 			}

test/e2e/benchmark_hack.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"testing"
+	"time"
+
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func readerPodYaml(size int) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		defer w.Close()
+		for i := 0; i < size; i++ {
+			_, _ = fmt.Fprintf(w, podYaml, i, i)
+		}
+	}()
+	return r
+}
+
+var podYaml = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-%d
+  namespace: default
+  uid: 00000000-0000-0000-0001-%012d
+spec:
+  containers:
+  - image: busybox
+    name: container-0
+  nodeName: node-0
+---
+`
+
+func scaleCreatePodWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-")
+	scaleCmd.Stdin = readerPodYaml(size)
+	if err := scaleCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start scale command: %w", err)
+	}
+
+	if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Running", size, 5, 10); err != nil {
+		return fmt.Errorf("failed to wait for resource: %w", err)
+	}
+	return nil
+}
+
+func readerPodDeleteYaml(size int) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		now := time.Now().UTC().Format(time.RFC3339)
+		defer w.Close()
+		for i := 0; i < size; i++ {
+			_, _ = fmt.Fprintf(w, podDeleteYaml, i, i, now)
+		}
+	}()
+	return r
+}
+
+var podDeleteYaml = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-%d
+  namespace: default
+  uid: 00000000-0000-0000-0001-%012d
+  deletionTimestamp: %s
+spec:
+  containers:
+  - image: busybox
+    name: container-0
+  nodeName: node-0
+---
+`
+
+func scaleDeletePodWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-")
+	scaleCmd.Stdin = readerPodDeleteYaml(size)
+	if err := scaleCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start scale command: %w", err)
+	}
+
+	if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Running", 0, 5, 10); err != nil {
+		return fmt.Errorf("failed to wait for resource: %w", err)
+	}
+	return nil
+}
+
+func readerNodeYaml(size int) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		defer w.Close()
+		for i := 0; i < size; i++ {
+			_, _ = fmt.Fprintf(w, nodeYaml, i, i)
+		}
+	}()
+	return r
+}
+
+var nodeYaml = `
+apiVersion: v1
+kind: Node
+metadata:
+  annotations:
+    kwok.x-k8s.io/node: fake
+    node.alpha.kubernetes.io/ttl: "0"
+  labels:
+    beta.kubernetes.io/arch: amd64
+    beta.kubernetes.io/os: linux
+    kubernetes.io/arch: amd64
+    kubernetes.io/os: linux
+    kubernetes.io/role: agent
+    node-role.kubernetes.io/agent: ""
+    type: kwok
+  name: node-%d
+  uid: 00000000-0000-0000-0000-%012d
+status:
+  allocatable:
+    cpu: "32"
+    memory: 256Gi
+    pods: "110"
+  capacity:
+    cpu: "32"
+    memory: 256Gi
+    pods: "110"
+---
+`
+
+func scaleCreateNodeWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-")
+	scaleCmd.Stdin = readerNodeYaml(size)
+	if err := scaleCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start scale command: %w", err)
+	}
+
+	if err := waitResource(ctx, t, kwokctlPath, name, "Node", "Ready", size, 10, 10); err != nil {
+		return fmt.Errorf("failed to wait for resource: %w", err)
+	}
+	return nil
+}
+
+func CaseBenchmarkWithHack(kwokctlPath, clusterName string) *features.FeatureBuilder {
+	return features.New("Benchmark").
+		Assess("Create nodes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			ctx0, cancel := context.WithTimeout(ctx, 180*time.Second)
+			defer cancel()
+
+			err := scaleCreateNodeWithHack(ctx0, t, kwokctlPath, clusterName, 5000)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).
+		Assess("Create pods", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			ctx0, cancel := context.WithTimeout(ctx, 240*time.Second)
+			defer cancel()
+
+			err := scaleCreatePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).
+		Assess("Delete pods", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			ctx0, cancel := context.WithTimeout(ctx, 240*time.Second)
+			defer cancel()
+
+			err := scaleDeletePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		})
+}

test/e2e/kwokctl/benchmark-hack/kubectl_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package benchmark_hack_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kwok/test/e2e"
+)
+
+func TestBenchmarkWithHack(t *testing.T) {
+	f0 := e2e.CaseBenchmarkWithHack(kwokctlPath, clusterName).
+		Feature()
+	testEnv.Test(t, f0)
+}

test/e2e/kwokctl/benchmark-hack/main_test.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package benchmark_hack_test is a test benchmarking environment for kwok.
+package benchmark_hack_test
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/support/kwok"
+
+	"sigs.k8s.io/kwok/pkg/consts"
+	"sigs.k8s.io/kwok/pkg/utils/path"
+	"sigs.k8s.io/kwok/test/e2e/helper"
+)
+
+var (
+	runtimeEnv  = consts.RuntimeTypeBinary
+	testEnv     env.Environment
+	pwd         = os.Getenv("PWD")
+	rootDir     = path.Join(pwd, "../../../..")
+	logsDir     = path.Join(rootDir, "logs")
+	clusterName = envconf.RandomName("kwok-e2e-benchmark-hack", 24)
+	kwokPath    = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwok"+helper.BinSuffix)
+	kwokctlPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwokctl"+helper.BinSuffix)
+	baseArgs    = []string{
+		"--kwok-controller-binary=" + kwokPath,
+		"--runtime=" + runtimeEnv,
+		"--wait=15m",
+		"--disable-kube-scheduler",
+		"--disable-qps-limits",
+	}
+)
+
+func init() {
+	_ = os.Setenv("KWOK_WORKDIR", path.Join(rootDir, "workdir"))
+}
+
+func TestMain(m *testing.M) {
+	testEnv = helper.Environment()
+
+	k := kwok.NewProvider().
+		WithName(clusterName).
+		WithPath(kwokctlPath)
+	testEnv.Setup(
+		helper.BuildKwokBinary(rootDir),
+		helper.BuildKwokctlBinary(rootDir),
+		helper.CreateCluster(k, baseArgs...),
+	)
+	testEnv.Finish(
+		helper.ExportLogs(k, logsDir),
+		helper.DestroyCluster(k),
+	)
+	os.Exit(testEnv.Run(m))
+}

test/e2e/kwokctl/benchmark/main_test.go

@@ -45,6 +45,7 @@ var (
 		"--runtime=" + runtimeEnv,
 		"--wait=15m",
 		"--disable-kube-scheduler",
+		"--disable-qps-limits",
 	}
 )