feat: introduce 'kvrocks-controller' to manage the cluster of kvrocks (…

…#58)
RocksLabs · Oct 23, 2023 · 0f74878 · 0f74878
1 parent cec2f4e
commit 0f74878
Show file tree

Hide file tree

Showing 43 changed files with 1,633 additions and 876 deletions.
diff --git a/api/v1alpha1/kvrocks_types.go b/api/v1alpha1/kvrocks_types.go
@@ -83,6 +83,7 @@ type KVRocksShrinkMsg struct {
 
 type KVRocksTopoPartitions struct {
 	PartitionName string            `json:"partitionName"`
+	Shard         int               `json:"shard"`
 	Topology      []KVRocksTopology `json:"topology"`
 }
 
@@ -95,18 +96,12 @@ type KVRocksTopology struct {
 	Slots    []string     `json:"slots,omitempty"`
 	MasterId string       `json:"masterId,omitempty"`
 	Migrate  []MigrateMsg `json:"migrate,omitempty"`
-	Import   []ImportMsg  `json:"import,omitempty"`
 	Failover bool         `json:"failover,omitempty"`
 }
 
-type ImportMsg struct {
-	SrcNode string   `json:"srcNode"`
-	Slots   []string `json:"slots"`
-}
-
 type MigrateMsg struct {
-	DstNode string   `json:"dstNode"`
-	Slots   []string `json:"slots"`
+	Shard int      `json:"shard"`
+	Slots []string `json:"slots"`
 }
 
 type KVRocksStorage struct {

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/kvrocks.apache.org_kvrocks.yaml b/config/crd/bases/kvrocks.apache.org_kvrocks.yaml
@@ -1024,40 +1024,28 @@ spec:
                   properties:
                     partitionName:
                       type: string
+                    shard:
+                      type: integer
                     topology:
                       items:
                         properties:
                           failover:
                             type: boolean
-                          import:
-                            items:
-                              properties:
-                                slots:
-                                  items:
-                                    type: string
-                                  type: array
-                                srcNode:
-                                  type: string
-                              required:
-                              - slots
-                              - srcNode
-                              type: object
-                            type: array
                           ip:
                             type: string
                           masterId:
                             type: string
                           migrate:
                             items:
                               properties:
-                                dstNode:
-                                  type: string
+                                shard:
+                                  type: integer
                                 slots:
                                   items:
                                     type: string
                                   type: array
                               required:
-                              - dstNode
+                              - shard
                               - slots
                               type: object
                             type: array
@@ -1084,6 +1072,7 @@ spec:
                       type: array
                   required:
                   - partitionName
+                  - shard
                   - topology
                   type: object
                 type: array

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -17,6 +17,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps.kruise.io
   resources:

diff --git a/deploy/crd/templates/crd.yaml b/deploy/crd/templates/crd.yaml
@@ -1024,40 +1024,28 @@ spec:
                   properties:
                     partitionName:
                       type: string
+                    shard:
+                      type: integer
                     topology:
                       items:
                         properties:
                           failover:
                             type: boolean
-                          import:
-                            items:
-                              properties:
-                                slots:
-                                  items:
-                                    type: string
-                                  type: array
-                                srcNode:
-                                  type: string
-                              required:
-                              - slots
-                              - srcNode
-                              type: object
-                            type: array
                           ip:
                             type: string
                           masterId:
                             type: string
                           migrate:
                             items:
                               properties:
-                                dstNode:
-                                  type: string
+                                shard:
+                                  type: integer
                                 slots:
                                   items:
                                     type: string
                                   type: array
                               required:
-                              - dstNode
+                              - shard
                               - slots
                               type: object
                             type: array
@@ -1084,6 +1072,7 @@ spec:
                       type: array
                   required:
                   - partitionName
+                  - shard
                   - topology
                   type: object
                 type: array

diff --git a/deploy/operator/templates/rbac.yaml b/deploy/operator/templates/rbac.yaml
@@ -167,6 +167,18 @@ rules:
       - patch
       - update
       - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/docs/design.md b/docs/design.md
@@ -2,32 +2,35 @@
 
 ## Sentinel
 
-![avatar](/docs/images/sentinel.png)
+<img alt="avatar" src="/docs/images/sentinel.png" width="50%"/>
 
 1. Use deployment to deploy sentinel Pod
 2. For scaling, just modify the spec.replicas field
 
 ### Expansion
 
-1. Add the spec.replicas field, but the number of replicas must be an odd number after the increase
+1. Add the `spec.replicas` field, but the number of replicas must be an odd number after the increase
 2. The new sentinel copy will automatically add the master information that the current sentinel has monitored
 3. Modify the number of quorum to (number of copies/2)+1
 
 ### Shrink
 
-1. Reduce the spec.replicas field, but the number of replicas after shrinking must be an odd number and must be greater than or equal to 3
+1. Reduce the spec.replicas field, but the number of replicas after shrinking must be an odd number and must be greater
+   than or equal to 3
 2. Modify the number of quorum to (number of copies/2)+1
 
 ### Fault Detection Recovery
 
 1. When an event of sentinel type is received, the following steps will be performed
-   - Determine whether the pods of the deployment are all in the Running state, if not, wait
-   - The operator starts a coroutine subscription +odown message for each sentinel for cluster mode failure detection and recovery
-   - Detect all pods with the label sentinel=xxx (xxx is the name of the current sentinel cluster), and add monitoring if the master ip changes or is not monitored.
+    - Determine whether the pods of the deployment are all in the Running state, if not, wait
+    - The operator starts a coroutine subscription +odown message for each sentinel for cluster mode failure detection
+      and recovery
+    - Detect all pods with the label sentinel=xxx (xxx is the name of the current sentinel cluster), and add monitoring
+      if the master ip changes or is not monitored.
 
 ## Standard
 
-![avatar](/docs/images/standard.png)
+<img src="/docs/images/standard.png" width="50%" />
 
 1. Use statefulSet to deploy kvrocks pod
 2. Sentinel is used to monitor kvrocks master-slave mode, and perform failover and discovery
@@ -37,12 +40,61 @@
 1. Clear sentinel's monitoring of the master before deleting the kvrocks instance
 2. Delete the kvrocks instance
 
-
 ### Fault Detection Recovery
 
 1. When an event of standard type is received, the following steps will be executed
-   - Detect whether the pods of the statefulSet are all in the Running state, if not waiting
-   - newly created kvrocks instance, slaveof myself on startup to make it a slave
-   - newly created kvrocks instance slaveof current master
-   - sentinel Check whether the monitoring information is correct, delete the old monitoring information incorrectly, and create a new one
+    - Detect whether the pods of the statefulSet are all in the Running state, if not waiting
+    - newly created kvrocks instance, slaveof myself on startup to make it a slave
+    - newly created kvrocks instance slaveof current master
+    - sentinel Check whether the monitoring information is correct, delete the old monitoring information incorrectly,
+      and create a new one
+
+## Cluster
+
+<img src="/docs/images/cluster.png" width="50%">
+
+### Apache Kvrocks Controller
+
+[Apache Kvrocks Controller](https://github.com/apache/kvrocks-controller) is a cluster management tool for Apache
+Kvrocks.
+
+1. Deploy the ETCD pod using a statefulSet. ETCD stores the cluster information.
+2. Deploy the Apache Kvrocks Controller pod using a deployment.
+
+### Apache Kvrocks Cluster
+
+1. Deploy the Apache Kvrocks Cluster pod using a statefulSet. Each statefulSet represents a shard of the cluster.
+2. Sentinel monitors the kvrocks master-slave mode, facilitating failover and discovery.
+
+### Expansion
+
+#### Expand Shard
+1. Add the `spec.master` field. However, the resulting number of replicas must be odd.
+2. The new shard will be added automatically, but the slots won't be rebalanced.
+
+#### Expand Nodes
+1. Add the `spec.replicas` field. However, the resulting number of replicas must be odd.
+2. The new node will join based on the shard, and the slots will be synchronized.
+
+### Shrink
 
+#### Shrink Shard
+1. **Important:** Before shrinking the shard, manually migrate the slots to other shards.
+2. Reduce the `spec.master` field. The resulting number of replicas must be an odd number and at least 3.
+3. The shard with the highest ID (or number) without slots will be deleted.
+
+
+#### Shrink Nodes
+1. Reduce the `spec.replicas` field. The resulting number of replicas must be an odd number and at least 1.
+2. Nodes with a `slave` role will be deleted.
+
+### Migration
+1. Use kubectl edit kvrocks xxxx to modify the kvrocks cluster.
+2. Add the content below to the master node of the shard you wish to migrate, and then save it.
+```yaml
+migrate:
+   - shard: 1   # the destination shard
+     slots:
+        - "1-2" # the slots to migrate
+        - "300"
+```
diff --git a/docs/images/cluster.png b/docs/images/cluster.png
diff --git a/docs/images/sentinel.png b/docs/images/sentinel.png
diff --git a/docs/images/standard.png b/docs/images/standard.png
diff --git a/examples/cluster.yaml b/examples/cluster.yaml
@@ -4,14 +4,13 @@ metadata:
   name: kvrocks-cluster-1-demo
   namespace: kvrocks
   labels:
-    kvrocks/system: xx
+    kvrocks/monitored-by: sentinel-1
 spec:
-  image: apache/kvrocks:nightly # kvrocks image
+  image: apache/kvrocks  # kvrocks image
   imagePullPolicy: IfNotPresent
   master: 3
-  replicas: 3
+  replicas: 2
   type: cluster
-  enableSentinel: true
   password: "123456"
   kvrocksConfig:
     bind: "0.0.0.0"
@@ -40,9 +39,9 @@ spec:
     rocksdb.compression: "no"
     rocksdb.wal_ttl_seconds: "0"
     rocksdb.wal_size_limit_mb: "0"
-#  storage:
-#    size: 32Gi
-#    class: xxxxx # storage class
+  # storage:
+  #   size: 32Gi
+  #   class: local-hostpath
 #  nodeSelector:
 #    role: kvrocks
   toleration:

diff --git a/examples/standard.yaml b/examples/standard.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     kvrocks/monitored-by: sentinel-1
 spec:
-  image: apache/kvrocks:nightly
+  image: apache/kvrocks
   imagePullPolicy: IfNotPresent
   master: 1
   replicas: 3

diff --git a/go.mod b/go.mod
@@ -7,7 +7,6 @@ require (
 	github.com/go-logr/logr v1.2.3
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/google/uuid v1.3.1
-	github.com/joaojeronimo/go-crc16 v0.0.0-20140729130949-59bd0194935e
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.6
 	github.com/openkruise/kruise-api v1.3.0
@@ -29,7 +28,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -61,11 +60,11 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/net v0.13.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect