Skip to content

Commit

Permalink
Merge branch 'main' into support/lorry_api_refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
xuriwuyun committed Oct 10, 2023
2 parents a19e24d + 2c51377 commit 1764682
Show file tree
Hide file tree
Showing 82 changed files with 378 additions and 171 deletions.
23 changes: 15 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -490,12 +490,15 @@ ifeq (, $(shell ls $(LOCALBIN)/kustomize 2>/dev/null))
endif

.PHONY: controller-gen
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
$(CONTROLLER_GEN): $(LOCALBIN)
ifeq (, $(shell ls $(LOCALBIN)/controller-gen 2>/dev/null))
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
endif

controller-gen: $(LOCALBIN) ## Download controller-gen locally if necessary.
@{ \
set -e ;\
if [ ! -f "$(CONTROLLER_GEN)" ] || [ "$$($(CONTROLLER_GEN) --version 2>&1 | awk '{print $$NF}')" != "$(CONTROLLER_TOOLS_VERSION)" ]; then \
echo 'Installing controller-gen@$(CONTROLLER_TOOLS_VERSION)...' ;\
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) ;\
echo 'Successfully installed' ;\
fi \
}

.PHONY: envtest
envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
Expand All @@ -504,7 +507,6 @@ ifeq (, $(shell ls $(LOCALBIN)/setup-envtest 2>/dev/null))
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
endif


.PHONY: install-docker-buildx
install-docker-buildx: ## Create `docker buildx` builder.
@if ! docker buildx inspect $(BUILDX_BUILDER) > /dev/null; then \
Expand All @@ -514,7 +516,6 @@ install-docker-buildx: ## Create `docker buildx` builder.
echo "Buildx builder $(BUILDX_BUILDER) already exists"; \
fi


.PHONY: golangci
golangci: GOLANGCILINT_VERSION = v1.51.2
golangci: ## Download golangci-lint locally if necessary.
Expand Down Expand Up @@ -664,6 +665,10 @@ else ifeq ($(TEST_TYPE), orioledb)
$(HELM) dependency build deploy/orioledb-cluster --skip-refresh
$(HELM) upgrade --install orioledb deploy/orioledb
$(HELM) template oriole-cluster deploy/orioledb-cluster > test/e2e/testdata/smoketest/orioledb/00_orioledbcluster.yaml
else ifeq ($(TEST_TYPE), weaviate)
$(HELM) dependency build deploy/weaviate-cluster --skip-refresh
$(HELM) upgrade --install weaviate deploy/weaviate
$(HELM) template weaviate-cluster deploy/weaviate-cluster > test/e2e/testdata/smoketest/weaviate/00_weaviatecluster.yaml
else
$(error "test type does not exist")
endif
Expand Down Expand Up @@ -707,6 +712,8 @@ else ifeq ($(TEST_TYPE), oceanbase)
$(HELM) upgrade --install official-postgresql deploy/official-postgresql
else ifeq ($(TEST_TYPE), openldap)
$(HELM) upgrade --install openldap deploy/openldap
else ifeq ($(TEST_TYPE), weaviate)
$(HELM) upgrade --install weaviate deploy/weaviate
else
$(error "test type does not exist")
endif
Expand Down
17 changes: 14 additions & 3 deletions controllers/apps/operations/volume_expansion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -160,7 +159,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
Count: 1,
Type: corev1.EventTypeWarning,
Reason: VolumeResizeFailed,
Message: "You've reached the maximum modification rate per volume limit. Wait at least 6 hours between modifications per EBS volume.",
Message: OptimisticLockErrorMsg,
}
stsInvolvedObject := corev1.ObjectReference{
Name: pvcNames[0],
Expand All @@ -175,12 +174,24 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() {
event.Count = 5
event.FirstTimestamp = metav1.Time{Time: time.Now()}
event.LastTimestamp = metav1.Time{Time: time.Now().Add(61 * time.Second)}

By("expect for progressDetail is not failed")
Expect(pvcEventHandler.Handle(k8sClient, reqCtx, eventRecorder, event)).Should(Succeed())
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) {
progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails
g.Expect(len(progressDetails) > 0).Should(BeTrue())
progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcNames[0]))
g.Expect(progressDetail.Status).ShouldNot(Equal(appsv1alpha1.FailedProgressStatus))
})).Should(Succeed())

By("expect for progressDetail is failed")
event.Message = "You've reached the maximum modification rate per volume limit. Wait at least 6 hours between modifications per EBS volume."
Expect(pvcEventHandler.Handle(k8sClient, reqCtx, eventRecorder, event)).Should(Succeed())
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) {
progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails
g.Expect(len(progressDetails) > 0).Should(BeTrue())
progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcNames[0]))
g.Expect(progressDetail.Status == appsv1alpha1.FailedProgressStatus).Should(BeTrue())
g.Expect(progressDetail.Status).Should(Equal(appsv1alpha1.FailedProgressStatus))
})).Should(Succeed())
}

Expand Down
6 changes: 6 additions & 0 deletions controllers/apps/operations/volume_expansion_updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package operations

import (
"strings"
"time"

"golang.org/x/exp/slices"
Expand All @@ -45,6 +46,7 @@ const (
VolumeResizeFailed = "VolumeResizeFailed"
// FileSystemResizeFailed the event reason of fileSystem resize failed on kubelet volume manager
FileSystemResizeFailed = "FileSystemResizeFailed"
OptimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)

func init() {
Expand Down Expand Up @@ -100,6 +102,10 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) Handle(cli client.Clien
return nil
}

// ignore the OptimisticLockErrorMsg
if strings.Contains(event.Message, OptimisticLockErrorMsg) {
return nil
}
// here, if the volume expansion ops is running, change the pvc status to Failed on the OpsRequest.
return pvcEventHandler.handlePVCFailedStatusOnRunningOpsRequests(cli, reqCtx, recorder, event, pvc)
}
Expand Down
27 changes: 25 additions & 2 deletions deploy/mongodb/templates/clusterversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-{{ default .Chart.AppVersion .Values.clusterVersionOverride }}
annotations:
kubeblocks.io/is-default-cluster-version: "true"
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
Expand Down Expand Up @@ -91,6 +89,31 @@ spec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.4

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-5.0.20
annotations:
kubeblocks.io/is-default-cluster-version: "true"
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
clusterDefinitionRef: mongodb
componentVersions:
- componentDefRef: mongodb
versionsContext:
containers:
- name: mongodb
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:5.0.20
imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:5.0.20
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:5.0.20

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
Expand Down
32 changes: 16 additions & 16 deletions deploy/postgresql/templates/clusterversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,18 @@ spec:
versionsContext:
initContainers:
- name: pg-init-container
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1-pgvector-v0.5.0
containers:
- name: postgresql
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1-pgvector-v0.5.0
- name: pgbouncer
image: {{ .Values.pgbouncer.image.registry | default "docker.io" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1-pgvector-v0.5.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1-pgvector-v0.5.0

---
apiVersion: apps.kubeblocks.io/v1alpha1
Expand Down Expand Up @@ -105,18 +105,18 @@ spec:
versionsContext:
initContainers:
- name: pg-init-container
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0-pgvector-v0.5.0
containers:
- name: postgresql
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0-pgvector-v0.5.0
- name: pgbouncer
image: {{ .Values.pgbouncer.image.registry | default "docker.io" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0-pgvector-v0.5.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.15.0-pgvector-v0.5.0

---
apiVersion: apps.kubeblocks.io/v1alpha1
Expand All @@ -134,18 +134,18 @@ spec:
versionsContext:
initContainers:
- name: pg-init-container
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0-pgvector-v0.5.0
containers:
- name: postgresql
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0-pgvector-v0.5.0
- name: pgbouncer
image: {{ .Values.pgbouncer.image.registry | default "docker.io" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0-pgvector-v0.5.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:14.8.0-pgvector-v0.5.0

---
# Adding backward compatibility for clusterVersion 12.14.0, which will be removed in subsequent iterative versions.
Expand Down Expand Up @@ -183,15 +183,15 @@ spec:
versionsContext:
initContainers:
- name: pg-init-container
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0-pgvector-v0.5.0
containers:
- name: postgresql
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0-pgvector-v0.5.0
- name: pgbouncer
image: {{ .Values.pgbouncer.image.registry | default "docker.io" }}/{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0-pgvector-v0.5.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0-pgvector-v0.5.0
2 changes: 1 addition & 1 deletion deploy/postgresql/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
image:
registry: registry.cn-hangzhou.aliyuncs.com
repository: apecloud/spilo
tag: 14.7.2
tag: 14.7.2-pgvector-v0.5.0
digest: ""
## Specify a imagePullPolicy
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: etcd-cluster-hscale-up-
generateName: etcd-cluster-hscale-out-
spec:
clusterRef: etcd-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: etcd-cluster-hscale-down-
generateName: etcd-cluster-hscale-in-
spec:
clusterRef: etcd-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: foxlake-cluster-hscale-up-
generateName: foxlake-cluster-hscale-out-
spec:
clusterRef: foxlake-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: foxlake-cluster-hscale-down-
generateName: foxlake-cluster-hscale-in-
spec:
clusterRef: foxlake-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: greptimedb-cluster-hscale-up-
generateName: greptimedb-cluster-hscale-in-
spec:
clusterRef: greptimedb-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: greptimedb-cluster-hscale-down-
generateName: greptimedb-cluster-hscale-out-
spec:
clusterRef: greptimedb-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: kafka-cluster-hscale-up-
generateName: kafka-cluster-hscale-out-
spec:
clusterRef: kafka-cluster
type: HorizontalScaling
Expand Down
18 changes: 18 additions & 0 deletions test/e2e/testdata/smoketest/kafka/06_reconfig.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: kafka-cluster-reconfiguring-
spec:
clusterRef: kafka-cluster
reconfigure:
componentName: broker
configurations:
- keys:
- key: kafka-configuration-tpl
parameters:
- key: MaxRAMPercentage
value: "200"
- key: MaxGCPauseMillis
value: "30"
name: mysql-consensusset-config
type: Reconfiguring
10 changes: 10 additions & 0 deletions test/e2e/testdata/smoketest/mongodb/06_hscale_out.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: mongodb-cluster-hscale-out-
spec:
clusterRef: mongodb-cluster
type: HorizontalScaling
horizontalScaling:
- componentName: mongodb
replicas: 3
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: kafka-cluster-hscale-down-
generateName: mongodb-cluster-hscale-in-
spec:
clusterRef: kafka-cluster
clusterRef: mongodb-cluster
type: HorizontalScaling
horizontalScaling:
- componentName: broker
replicas: 2
- componentName: mongodb
replicas: 1
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: nebula-cluster-hscale-up-
generateName: nebula-cluster-hscale-out-
spec:
clusterRef: nebula-cluster
type: HorizontalScaling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: apps.kubeblocks.io/v1alpha1
kind: OpsRequest
metadata:
generateName: nebula-cluster-hscale-down-
generateName: nebula-cluster-hscale-in-
spec:
clusterRef: nebula-cluster
type: HorizontalScaling
Expand Down
Loading

0 comments on commit 1764682

Please sign in to comment.