From 5f6822bb236062cddb1249d9a7ccce0379b501b2 Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 26 Sep 2023 17:07:40 +0800 Subject: [PATCH 1/4] create and delete works --- deploy/etcd-cluster/templates/NOTES.txt | 22 +------ deploy/etcd-cluster/templates/ingress.yaml | 61 ------------------- .../templates/tests/test-connection.yaml | 15 ----- deploy/etcd/templates/clusterdefinition.yaml | 47 +++++++++++--- .../smoketest/etcd/00_etcdcluster.yaml | 21 ------- 5 files changed, 39 insertions(+), 127 deletions(-) delete mode 100644 deploy/etcd-cluster/templates/ingress.yaml delete mode 100644 deploy/etcd-cluster/templates/tests/test-connection.yaml diff --git a/deploy/etcd-cluster/templates/NOTES.txt b/deploy/etcd-cluster/templates/NOTES.txt index 57f61e32e51..786eacaf78b 100644 --- a/deploy/etcd-cluster/templates/NOTES.txt +++ b/deploy/etcd-cluster/templates/NOTES.txt @@ -1,22 +1,2 @@ 1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "etcd-cluster.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "etcd-cluster.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "etcd-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "etcd-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +kubectl get svc --namespace {{ .Release.Namespace }} diff --git a/deploy/etcd-cluster/templates/ingress.yaml b/deploy/etcd-cluster/templates/ingress.yaml deleted file mode 100644 index 551983970e3..00000000000 --- a/deploy/etcd-cluster/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "etcd-cluster.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "etcd-cluster.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deploy/etcd-cluster/templates/tests/test-connection.yaml b/deploy/etcd-cluster/templates/tests/test-connection.yaml deleted file mode 100644 index f6bbd83c2cd..00000000000 --- a/deploy/etcd-cluster/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "clustername" . }}-test-connection" - labels: - {{- include "etcd-cluster.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "clustername" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index 4b3024abcec..f60f029470a 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -10,18 +10,47 @@ spec: - name: etcd characterType: etcd workloadType: Consensus - consensusSpec: - leader: - name: "leader" - accessMode: ReadWrite - followers: + rsmSpec: + roles: + - name: "leader" + accessMode: ReadWrite + isLeader: true + canVote: true - name: "follower" accessMode: ReadWrite - updateStrategy: BestEffortParallel - probes: + isLeader: false + canVote: true roleProbe: - periodSeconds: 1 - failureThreshold: 3 + probeActions: + - image: "quay.io/coreos/etcd:v3.5.6" + command: + - "Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) &&" + - "IsLeader=$(echo $Status | awk -F ', ' '{print $5}') &&" + - "IsLearner=$(echo $Status | awk -F ', ' '{print $6}') &&" + - "if [ \"true\" = \"$IsLeader\" ]; then echo -n \"leader\"; elif [ \"true\" = \"$IsLearner\" ]; then echo -n \"learner\"; else echo -n \"follower\"; fi" + roleUpdateMechanism: DirectAPIServerEventUpdate + memberUpdateStrategy: BestEffortParallel + membershipReconfiguration: + switchoverAction: + image: "quay.io/coreos/etcd:v3.5.6" + command: + - "current_leader=$(etcdctl endpoint status --write-out=json | jq -r '.[] | select(.is_leader == true) | .clientURLs[0]') &&" + - "[[ -z $current_leader ]] && echo \"No current leader found. Exiting...\" && exit 1 &&" + - "echo \"Current Leader: $current_leader\" &&" + - "follower_id=$(etcdctl member list | awk '/isLeader=False/{print $1}') &&" + - "[[ -z $follower_id ]] && echo \"No follower found. Exiting...\" && exit 1 &&" + - "echo \"Promoting Follower: $follower_id\" &&" + - "etcdctl member promote $follower_id &&" + - "new_leader=$(etcdctl endpoint status --write-out=json | jq -r '.[] | select(.is_leader == true) | .clientURLs[0]') &&" + - "[[ $new_leader == $current_leader ]] && echo \"Switchover failed. New leader is the same as the previous leader.\" && exit 1 &&" + - "echo \"New Leader: $new_leader\" &&" + - "echo \"Etcd Switchover completed successfully.\"" + memberJoinAction: + command: + - "etcdctl member add --peer-urls=http://$KB_RSM_TARGET_HOST:KB_RSM_SERVICE_PORT" + memberLeaveAction: + command: + - "etcdctl member remove --peer-urls=http://$KB_RSM_TARGET_HOST:KB_RSM_SERVICE_PORT" service: ports: - name: client diff --git a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml index 0d08c843984..d1d3fd0022a 100644 --- a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml +++ b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml @@ -70,24 +70,3 @@ spec: monitor: false replicas: 3 serviceAccountName: kb-etcd-cluster ---- -# Source: etcd-cluster/templates/tests/test-connection.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "etcd-cluster-test-connection" - labels: - helm.sh/chart: etcd-cluster-0.1.0 - app.kubernetes.io/name: etcd-cluster - app.kubernetes.io/instance: etcd-cluster - app.kubernetes.io/version: "v3.5.6" - app.kubernetes.io/managed-by: Helm - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['etcd-cluster:'] - restartPolicy: Never From f41bbb32aba5a3dc4a966ead508a6fe51c31af0c Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 27 Sep 2023 14:44:47 +0800 Subject: [PATCH 2/4] role probe works, member join&leave not working yet --- deploy/etcd/templates/clusterdefinition.yaml | 51 ++++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index f60f029470a..1bedd490869 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -12,45 +12,54 @@ spec: workloadType: Consensus rsmSpec: roles: - - name: "leader" + - name: leader accessMode: ReadWrite isLeader: true canVote: true - - name: "follower" + - name: follower accessMode: ReadWrite isLeader: false canVote: true roleProbe: probeActions: - - image: "quay.io/coreos/etcd:v3.5.6" + - image: quay.io/coreos/etcd:v3.5.6 command: - - "Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) &&" - - "IsLeader=$(echo $Status | awk -F ', ' '{print $5}') &&" - - "IsLearner=$(echo $Status | awk -F ', ' '{print $6}') &&" - - "if [ \"true\" = \"$IsLeader\" ]; then echo -n \"leader\"; elif [ \"true\" = \"$IsLearner\" ]; then echo -n \"learner\"; else echo -n \"follower\"; fi" + - | + Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) && + IsLeader=$(echo $Status | awk -F ', ' '{print $5}') && + IsLearner=$(echo $Status | awk -F ', ' '{print $6}') && + if [ "true" = "$IsLeader" ]; then echo -n "leader"; elif [ "true" = "$IsLearner" ]; then echo -n "learner"; else echo -n "follower"; fi roleUpdateMechanism: DirectAPIServerEventUpdate memberUpdateStrategy: BestEffortParallel membershipReconfiguration: switchoverAction: - image: "quay.io/coreos/etcd:v3.5.6" + image: quay.io/coreos/etcd:v3.5.6 command: - - "current_leader=$(etcdctl endpoint status --write-out=json | jq -r '.[] | select(.is_leader == true) | .clientURLs[0]') &&" - - "[[ -z $current_leader ]] && echo \"No current leader found. Exiting...\" && exit 1 &&" - - "echo \"Current Leader: $current_leader\" &&" - - "follower_id=$(etcdctl member list | awk '/isLeader=False/{print $1}') &&" - - "[[ -z $follower_id ]] && echo \"No follower found. Exiting...\" && exit 1 &&" - - "echo \"Promoting Follower: $follower_id\" &&" - - "etcdctl member promote $follower_id &&" - - "new_leader=$(etcdctl endpoint status --write-out=json | jq -r '.[] | select(.is_leader == true) | .clientURLs[0]') &&" - - "[[ $new_leader == $current_leader ]] && echo \"Switchover failed. New leader is the same as the previous leader.\" && exit 1 &&" - - "echo \"New Leader: $new_leader\" &&" - - "echo \"Etcd Switchover completed successfully.\"" + - /bin/bash + - -c + - | + echo "Prepare to promote $KB_RSM_TARGET_HOST" && + follower_id=$(etcdctl --endpoints="http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT" member list | grep "$KB_RSM_TARGET_HOST" | awk -F',' '{print $1}') && + [[ -z $follower_id ]] && echo "No follower found. Exiting..." && exit 1 && + echo "Promoting Follower: $follower_id" && + etcdctl --endpoints=http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT member promote $follower_id && + new_leader=$(etcdctl --endpoints=http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT endpoint status --cluster | awk '{print $2,$6}' | grep 'true' | awk -F',' '{print $1}') && + [[ $new_leader != $follower_id ]] && echo "Switchover failed. New leader is not the same as expected." && exit 1 && + echo "New Leader: $new_leader" && + echo "Etcd Switchover completed successfully." memberJoinAction: command: - - "etcdctl member add --peer-urls=http://$KB_RSM_TARGET_HOST:KB_RSM_SERVICE_PORT" + - /bin/bash + - -c + - | + etcdctl --endpoints=http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT member add $KB_RSM_TARGET_HOST --peer-urls="http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT" memberLeaveAction: command: - - "etcdctl member remove --peer-urls=http://$KB_RSM_TARGET_HOST:KB_RSM_SERVICE_PORT" + - /bin/bash + - -c + - | + member_id=$(etcdctl --endpoints=http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT member list | grep "$KB_RSM_TARGET_HOST" | awk -F',' '{print $1}') && + etcdctl member remove $member_id service: ports: - name: client From e045177899cfaba57c852ccf9828644e38763fab Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 27 Sep 2023 14:52:40 +0800 Subject: [PATCH 3/4] stop&start works, vscale, restart, hscale not yet --- deploy/etcd/templates/clusterdefinition.yaml | 29 ------------------- .../testdata/smoketest/etcd/03_vscale.yaml | 6 ++-- 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index 1bedd490869..fe7078130c2 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -31,35 +31,6 @@ spec: if [ "true" = "$IsLeader" ]; then echo -n "leader"; elif [ "true" = "$IsLearner" ]; then echo -n "learner"; else echo -n "follower"; fi roleUpdateMechanism: DirectAPIServerEventUpdate memberUpdateStrategy: BestEffortParallel - membershipReconfiguration: - switchoverAction: - image: quay.io/coreos/etcd:v3.5.6 - command: - - /bin/bash - - -c - - | - echo "Prepare to promote $KB_RSM_TARGET_HOST" && - follower_id=$(etcdctl --endpoints="http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT" member list | grep "$KB_RSM_TARGET_HOST" | awk -F',' '{print $1}') && - [[ -z $follower_id ]] && echo "No follower found. Exiting..." && exit 1 && - echo "Promoting Follower: $follower_id" && - etcdctl --endpoints=http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT member promote $follower_id && - new_leader=$(etcdctl --endpoints=http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT endpoint status --cluster | awk '{print $2,$6}' | grep 'true' | awk -F',' '{print $1}') && - [[ $new_leader != $follower_id ]] && echo "Switchover failed. New leader is not the same as expected." && exit 1 && - echo "New Leader: $new_leader" && - echo "Etcd Switchover completed successfully." - memberJoinAction: - command: - - /bin/bash - - -c - - | - etcdctl --endpoints=http://$KB_RSM_LEADER_HOST:$KB_RSM_SERVICE_PORT member add $KB_RSM_TARGET_HOST --peer-urls="http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT" - memberLeaveAction: - command: - - /bin/bash - - -c - - | - member_id=$(etcdctl --endpoints=http://$KB_RSM_TARGET_HOST:$KB_RSM_SERVICE_PORT member list | grep "$KB_RSM_TARGET_HOST" | awk -F',' '{print $1}') && - etcdctl member remove $member_id service: ports: - name: client diff --git a/test/e2e/testdata/smoketest/etcd/03_vscale.yaml b/test/e2e/testdata/smoketest/etcd/03_vscale.yaml index 5c0cb2e0a3e..f5a8fd169e8 100644 --- a/test/e2e/testdata/smoketest/etcd/03_vscale.yaml +++ b/test/e2e/testdata/smoketest/etcd/03_vscale.yaml @@ -7,6 +7,6 @@ spec: type: VerticalScaling verticalScaling: - componentName: etcd - requests: - cpu: "500m" - memory: 500Mi \ No newline at end of file + requests: + cpu: "500m" + memory: 500Mi From 9ead759842de5c7f4acaf2de3d2aeb45f4c028c1 Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 31 Oct 2023 19:34:09 +0800 Subject: [PATCH 4/4] update crash fixed --- deploy/etcd-cluster/values.yaml | 4 ++-- deploy/etcd/templates/clusterdefinition.yaml | 5 ++--- test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml | 9 +++++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/deploy/etcd-cluster/values.yaml b/deploy/etcd-cluster/values.yaml index 47ff849f7e7..b1942938125 100644 --- a/deploy/etcd-cluster/values.yaml +++ b/deploy/etcd-cluster/values.yaml @@ -57,7 +57,7 @@ resources: { } persistence: ## @param shard[*].persistence.enabled Enable persistence using Persistent Volume Claims ## - enabled: false + enabled: true ## `data` volume settings ## data: @@ -71,7 +71,7 @@ persistence: storageClassName: ## @param shard[*].persistence.size Size of data volume ## - size: 10Gi + size: 1Gi ingress: ## @param ingress.enabled Enable ingress record generation for etcd diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index bb472677a9e..5fa41b864b0 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -21,7 +21,7 @@ spec: isLeader: false canVote: true roleProbe: - probeActions: + customHandler: - image: quay.io/coreos/etcd:v3.5.6 command: - | @@ -70,13 +70,12 @@ spec: # TODO: clusterDomain 'cluster.local' requires configurable MY_PEER=$KB_POD_FQDN{{ .Values.clusterDomain }} exec etcd --name ${HOSTNAME} \ + --experimental-initial-corrupt-check=true \ --listen-peer-urls http://0.0.0.0:2380 \ --listen-client-urls http://0.0.0.0:2379 \ --advertise-client-urls http://${MY_PEER}:2379 \ --initial-advertise-peer-urls http://${MY_PEER}:2380 \ - --initial-cluster-token etcd-cluster-1 \ --initial-cluster ${PEERS} \ - --initial-cluster-state new \ --data-dir /var/run/etcd/default.etcd connectionCredential: username: root diff --git a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml index 0a93c1afd69..fa5facd6e00 100644 --- a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml +++ b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml @@ -23,3 +23,12 @@ spec: monitor: false replicas: 3 serviceAccountName: kb-etcd-cluster + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi