Add rollout restart agent e2e test

.github/actions/setup-e2e/action.yaml

@@ -32,6 +32,10 @@ inputs:
     description: "If k3d is not required, set this to false"
     required: false
     default: "true"
+  require_k3d_storage:
+    description: "If local-path-storage is not required, set this to false"
+    required: false
+    default: "false"
   require_minikube:
     description: "If minikube is not required, set this to true and set require_k3d to false"
     required: false
@@ -85,6 +89,7 @@ runs:
       with:
         agents: 3
         ingress_port: ${{ inputs.ingress_port }}
+        storage: ${{ inputs.require_k3d_storage }}
     - name: Setup Minikube environment
       if: ${{ inputs.require_minikube == 'true' }}
       shell: bash

.github/actions/setup-k3d/action.yaml

@@ -36,6 +36,10 @@ inputs:
     description: "Number of agents"
     required: false
     default: "3"
+  storage:
+    description: "If true, the local-path-storage will be deployed"
+    required: false
+    default: "false"
   options:
     description: "Options for k3d cluster create command"
     required: false
@@ -124,6 +128,11 @@ runs:
         echo $KUBECONFIG
         cat $KUBECONFIG
         cat /etc/hosts
+    - name: Set local path storage
+      if: ${{ inputs.storage == 'true' }}
+      shell: bash
+      run: |
+        make k3d/storage
     - name: Show Kubernetes Cluster Info
       shell: bash
       run: |

.github/helm/values/values-rollout-agent.yaml

@@ -0,0 +1,138 @@
+#
+# Copyright (C) 2019-2025 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  image:
+    tag: v1.7.16
+  logging:
+    level: debug
+  server_config:
+    metrics:
+      pprof:
+        enabled: true # enable Pyroscope
+    healths:
+      liveness:
+        livenessProbe:
+          initialDelaySeconds: 60
+      readiness:
+        readinessProbe:
+          initialDelaySeconds: 60
+    servers:
+      grpc:
+        server:
+          grpc:
+            interceptors:
+              - RecoverInterceptor
+              - TraceInterceptor
+              - MetricInterceptor
+  grpc:
+    client:
+      dial_option:
+        interceptors:
+          - TraceInterceptor
+  observability:
+    enabled: true
+    otlp:
+      collector_endpoint: "opentelemetry-collector-collector.default.svc.cluster.local:4317"
+    trace:
+      enabled: true
+  networkPolicy:
+    enabled: true
+    custom:
+      ingress:
+        - from:
+            - podSelector:
+                matchLabels:
+                  app.kubernetes.io/name: pyroscope
+      egress:
+        - to:
+            - podSelector:
+                matchLabels:
+                  app.kubernetes.io/name: opentelemetry-collector-collector
+gateway:
+  lb:
+    enabled: true
+    maxReplicas: 1
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 200m
+        memory: 150Mi
+    gateway_config:
+      index_replica: 2
+      # ingress:
+      #   # if enabled is true, vald-lb-gateway can be connected through Kubernetes ingress from the external network.
+      #   enabled: true
+      #   # TODO: Set your ingress host.
+      #   host: localhost
+      # service:
+      #   # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service
+      #   annotations:
+      #     traefik.ingress.kubernetes.io/service.serversscheme: h2c
+agent:
+  minReplicas: 3
+  maxReplicas: 3
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  # We recommend to set this value long enough to ensure the backup speed of PV, since the Index is backed up at the end of the pod.
+  terminationGracePeriodSeconds: 600
+  # This is the persistent volume settings.
+  # Please change it according to your environment.
+  persistentVolume:
+    enabled: true
+    accessMode: ReadWriteOncePod
+    storageClass: local-path
+    size: 500Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 500
+    dimension: 784
+    enable_in_memory_mode: false
+    index_path: "/var/ngt/index"
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+manager:
+  index:
+    enabled: true
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
+    corrector:
+      enabled: false
+      # suspend because you do not want corrector to start automatically in CI
+      # instead run it manually
+      suspend: true
+      schedule: "1 2 3 4 5"

.github/workflows/dockers-agent-faiss-image.yaml

@@ -265,4 +265,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-faiss
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-agent-image.yaml

@@ -71,4 +71,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-agent-ngt-image.yaml

@@ -269,4 +269,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-ngt
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-agent-sidecar-image.yaml

@@ -295,4 +295,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-sidecar
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-benchmark-job-image.yaml

@@ -261,4 +261,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: benchmark-job
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-benchmark-operator-image.yaml

@@ -255,4 +255,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: benchmark-operator
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-dev-container-image.yaml

@@ -55,4 +55,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: dev-container
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-discoverer-k8s-image.yaml

@@ -259,4 +259,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: discoverer-k8s
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-example-client-image.yaml

@@ -65,4 +65,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: example-client
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-gateway-filter-image.yaml

@@ -259,4 +259,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: gateway-filter
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-gateway-lb-image.yaml

@@ -257,4 +257,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: gateway-lb
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-gateway-mirror-image.yaml

@@ -261,4 +261,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: gateway-mirror
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-helm-operator-image.yaml

@@ -65,4 +65,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: helm-operator
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-index-correction-image.yaml

@@ -249,4 +249,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: index-correction
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-index-creation-image.yaml

@@ -243,4 +243,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: index-creation
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-index-deletion-image.yaml

@@ -243,4 +243,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: index-deletion
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-index-operator-image.yaml

@@ -241,4 +241,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: index-operator
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-index-save-image.yaml

@@ -243,4 +243,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: index-save
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-manager-index-image.yaml

@@ -263,4 +263,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: manager-index
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-readreplica-rotate-image.yaml

@@ -239,4 +239,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: readreplica-rotate
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/e2e.yaml

@@ -348,6 +348,49 @@ jobs:
             e2e/readreplica
         env:
           POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }}
+  e2e-stream-crud-with-rollout-restart-agent:
+    name: "E2E test (Stream CRUD) with rollout restart agent"
+    needs: [detect-ci-container]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Setup E2E environment
+        id: setup_e2e
+        uses: ./.github/actions/setup-e2e
+        with:
+          require_k3d_storage: true
+      - name: Deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        with:
+          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-rollout-agent.yaml
+          wait_for_selector: app=vald-lb-gateway
+      - name: Run E2E CRUD with rollout restart agent
+        run: |
+          make hack/benchmark/assets/dataset/${{ env.DATASET }}
+          make E2E_BIND_PORT=8081 \
+            E2E_DATASET_NAME=${{ env.DATASET }} \
+            E2E_INSERT_COUNT=1000 \
+            E2E_SEARCH_COUNT=1000 \
+            E2E_SEARCH_BY_ID_COUNT=1000 \
+            E2E_GET_OBJECT_COUNT=100 \
+            E2E_UPDATE_COUNT=100 \
+            E2E_UPSERT_COUNT=100 \
+            E2E_REMOVE_COUNT=100 \
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_TARGET_POD_NAME=${POD_NAME} \
+            E2E_TARGET_NAMESPACE=default \
+            e2e/rollout/restart/agent
+        env:
+          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-with-mirror:
     name: "E2E test (Stream CRUD) with mirror"
     needs: [detect-ci-container]
@@ -417,6 +460,7 @@ jobs:
       - e2e-stream-crud-under-index-management-jobs
       - e2e-stream-crud-with-mirror
       - e2e-stream-crud-with-readreplica
+      - e2e-stream-crud-with-rollout-restart-agent
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

Makefile

@@ -359,6 +359,7 @@ E2E_PORTFORWARD_ENABLED            ?= true
 E2E_REMOVE_COUNT                   ?= 3
 E2E_SEARCH_BY_ID_COUNT             ?= 100
 E2E_SEARCH_COUNT                   ?= 1000
+E2E_SEARCH_CONCURRENCY             ?= 10
 E2E_TARGET_NAME                    ?= vald-lb-gateway
 E2E_TARGET_NAMESPACE               ?= default
 E2E_TARGET_POD_NAME                ?= $(eval E2E_TARGET_POD_NAME := $(shell kubectl get pods --selector=app=$(E2E_TARGET_NAME) -n $(E2E_TARGET_NAMESPACE) | tail -1 | cut -f1 -d " "))$(E2E_TARGET_POD_NAME)
@@ -368,6 +369,7 @@ E2E_UPDATE_COUNT                   ?= 10
 E2E_UPSERT_COUNT                   ?= 10
 E2E_WAIT_FOR_CREATE_INDEX_DURATION ?= 8m
 E2E_WAIT_FOR_START_TIMEOUT         ?= 10m
+E2E_WAIT_FOR_RESOURCE_READY        ?= 3m
 E2E_SEARCH_FROM                    ?= 0
 E2E_SEARCH_BY_ID_FROM              ?= 0
 E2E_INSERT_FROM                    ?= 0

Makefile.d/e2e.mk

@@ -89,6 +89,16 @@ e2e/index/job/correction:
 e2e/readreplica:
 	$(call run-e2e-crud-test,-run TestE2EReadReplica)
 
+.PHONY: e2e/rollaout/restart/agent
+## run rollout-restart agent e2e
+e2e/rollout/restart/agent:
+	$(call run-e2e-crud-test,-run TestE2EAgentRolloutRestart)
+
+.PHONY: e2e/large/search
+## run big multisearch agent e2e
+e2e/large/search:
+	$(call run-e2e-crud-test,-run TestE2EHighConcurrencyMultiSearch)
+
 .PHONY: e2e/maxdim
 ## run e2e/maxdim
 e2e/maxdim: