Finish pod-rebalancer-v0.0.1

.devcontainer/work/microk8s/pod-rebalancer.yaml

@@ -0,0 +1,64 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pod-rebalancer
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs:     ["list","delete","get"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs:     ["list","get"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs:     ["list","get"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: pod-rebalancer
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pod-rebalancer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pod-rebalancer
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: pod-rebalancer
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  namespace: kube-system
+  name: pod-rebalancer
+spec:
+  schedule: "*/3 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: pod-rebalancer
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                    - amd64
+          containers:
+          - name: pod-rebalancer
+            image: localhost:32000/norseto/pod-rebalancer:latest
+            imagePullPolicy: IfNotPresent
+          restartPolicy: OnFailure
+          activeDeadlineSeconds: 60
+---

.gitignore

@@ -42,7 +42,8 @@ $RECYCLE.BIN/
 .LSOverride
 
 # Icon must end with two \r
-Icon
+Icon
+
 
 # Thumbnails
 ._*
@@ -63,4 +64,10 @@ Network Trash Folder
 Temporary Items
 .apdisk
 
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/Readme.md
 

.vscode/Readme.md

@@ -0,0 +1,7 @@
+### Launch configuration
+---
+Launch configuration "Microk8s" in launch.json requires microk8s.config file in this directory. To create this file,
+1. Run `kubectl config view --raw` and save as microk8s.config.
+1. Run `multipass list` and get microk8s IP address.
+1. Rewite server IP address in microk8s.config.
+

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Microk8s",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "env": {},
+            "args": ["--kubeconfig", "/workspaces/k8s-watchdogs/.vscode/microk8s.config"]
+        }
+    ]
+}

README.md

@@ -8,3 +8,14 @@ This CronJos cleans "Evicted" pods.
 ```
 kubectl apply -f https://github.com/norseto/k8s-watchdogs/releases/download/evicted-cleaner-v0.1.0/evicted-cleaner.yaml
 ```
+
+## Pod Rebalancer
+Delete a pod that is scheduled to be biased to a specific node.
+
+### Installation
+```
+kubectl apply -f https://github.com/norseto/k8s-watchdogs/releases/download/pod-rebalancer-v0.0.1/pod-rebalancer.yaml
+```
+
+### Limitation
+Ignores pods with affinity or tolerations.

cmd/pod-rebalancer/Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.13-alpine as BUILD
+
+RUN mkdir -p /build
+RUN mkdir /dist
+WORKDIR /build
+
+ENV CGO_ENABLED=0
+COPY . /build/
+RUN go get github.com/Songmu/gocredits/cmd/gocredits \
+	&& go mod download \
+	&& go vet cmd/pod-rebalancer/*.go \
+	&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /build/pod-rebalancer cmd/pod-rebalancer/*.go \
+	&& gocredits > /dist/CREDITS \
+	&& cp pod-rebalancer /dist \
+	&& cp LICENSE /dist \
+	;
+
+FROM scratch
+WORKDIR /
+COPY --from=BUILD /dist /
+CMD ["/pod-rebalancer"]

cmd/pod-rebalancer/main.go

@@ -0,0 +1,122 @@
+package main
+
+// Evicted Pod Cleaner
+// Deletes all evicted pod.
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/norseto/k8s-watchdogs/pkg/k8sutils"
+)
+
+func main() {
+	var clientset *kubernetes.Clientset
+	var namespace = metav1.NamespaceAll
+
+	log.Info("Starging multiple pod rs rebalancer...")
+
+	clientset, err := k8sutils.NewClientset()
+	if err != nil {
+		log.Panic(errors.Wrap(err, "failed to create clientset"))
+	}
+
+	nodes, err := k8sutils.GetUntaintedNodes(clientset)
+	if err != nil {
+		log.Panic(errors.Wrap(err, "failed to list nodes"))
+	}
+
+	replicasets, err := getTargetReplicasets(clientset, namespace)
+	if err != nil {
+		log.Panic(errors.Wrap(err, "failed to list replicaset"))
+	}
+	rs, err := getTargetPods(clientset, namespace, nodes, replicasets)
+	if err != nil {
+		log.Panic(errors.Wrap(err, "failed to list pods"))
+	}
+
+	if len(rs) < 1 {
+		log.Info("No rs. Do nothing.")
+		return
+	}
+
+	rebalanced := 0
+	for _, r := range rs {
+		name := r.replicaset.Name
+		result, err := newRebalancer(r).Rebalance(clientset)
+		if err != nil {
+			log.Error(errors.Wrap(err, fmt.Sprint("failed to rebalance rs: ", name)))
+		} else if result {
+			log.Debug(fmt.Sprint("Rebalanced rs: ", name))
+			rebalanced++
+		} else {
+			log.Debug(fmt.Sprint("No need to rebalance rs: ", name))
+		}
+	}
+
+	log.Info("Done multiple pod rs rebalancer. Rebalanced ", rebalanced, " ReplicaSet(s)")
+}
+
+// getTargetReplicasets gets target replicaset.
+// Parameter:
+//   c *kubernetes.Clientset : clientset
+//   ns string : namespace of replicaset
+// Returns:
+//   []appsv1.ReplicaSet : All target replicasets that does not hace
+//                         affinity nor tolerations nor nodeselector
+//   error : error if error happens
+func getTargetReplicasets(c *kubernetes.Clientset, ns string) ([]appsv1.ReplicaSet, error) {
+	var replicasets = []appsv1.ReplicaSet{}
+	all, err := c.AppsV1().ReplicaSets(ns).List(metav1.ListOptions{IncludeUninitialized: false})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list replicaset")
+	}
+	for _, rs := range all.Items {
+		replicasets = append(replicasets, rs)
+	}
+	return replicasets, nil
+}
+
+// getTargetPods gets target pods.
+func getTargetPods(c *kubernetes.Clientset, ns string, nodes []v1.Node, rslist []appsv1.ReplicaSet) ([]*replicaState, error) {
+	nodeMap := make(map[string]v1.Node)
+	stats := []*replicaState{}
+	rsmap := make(map[types.UID]*replicaState)
+
+	for _, n := range nodes {
+		nodeMap[n.Name] = n
+	}
+
+	pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{IncludeUninitialized: false})
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprint("failed to list pod for ", ns))
+	}
+	for _, po := range pods.Items {
+		if !k8sutils.IsPodReadyRunning(po) {
+			continue
+		}
+		for _, rs := range rslist {
+			if !k8sutils.IsPodOwnedBy(rs, po) {
+				continue
+			}
+			node := nodeMap[po.Spec.NodeName]
+			postat := podState{pod: &po, node: &node}
+			rstat, ok := rsmap[rs.ObjectMeta.UID]
+			if !ok {
+				rstat = &replicaState{replicaset: &rs, nodes: nodes}
+				rsmap[rs.ObjectMeta.UID] = rstat
+				stats = append(stats, rstat)
+			}
+			rstat.podState = append(rstat.podState, postat)
+			break
+		}
+	}
+	return stats, nil
+}

cmd/pod-rebalancer/rebalancer.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/norseto/k8s-watchdogs/pkg/k8sutils"
+)
+
+type replicaState struct {
+	replicaset *appsv1.ReplicaSet
+	nodes      []v1.Node
+	podState   []podState
+}
+
+type podState struct {
+	pod  *v1.Pod
+	node *v1.Node
+}
+
+type rebalancer struct {
+	current *replicaState
+}
+
+func (r *rebalancer) specReplicas() int32 {
+	return *r.current.replicaset.Spec.Replicas
+}
+
+func (r *rebalancer) currentReplicas() int32 {
+	return r.current.replicaset.Status.Replicas
+}
+
+func newRebalancer(current *replicaState) *rebalancer {
+	return &rebalancer{current: current}
+}
+
+func (r *rebalancer) Rebalance(c *kubernetes.Clientset) (bool, error) {
+	nodeCount := len(r.current.nodes)
+	rs := r.current.replicaset
+	sr := r.specReplicas()
+
+	if nodeCount < 2 || sr < 2 || r.currentReplicas() < sr ||
+		k8sutils.IsPodScheduleLimeted(*rs) {
+		return false, nil
+	}
+
+	node, num := r.maxPodNode()
+	ave := float32(sr) / float32(nodeCount)
+	if len(node) > 0 && num >= int(ave+1.0) {
+		err := r.deleteNodePod(c, node)
+		return true, err
+	}
+
+	return false, nil
+}
+
+// deleteNodePod deletes only one pod per replicaset.
+func (r *rebalancer) deleteNodePod(c *kubernetes.Clientset, node string) error {
+	for _, s := range r.current.podState {
+		if s.node.Name == node {
+			return k8sutils.DeletePod(c, *s.pod)
+		}
+	}
+	return nil
+}
+
+func (r *rebalancer) maxPodNode() (string, int) {
+	m := map[string]int{}
+	for _, n := range r.current.nodes {
+		m[n.Name] = 0
+	}
+	for _, s := range r.current.podState {
+		m[s.node.Name]++
+	}
+
+	maxVal := 0
+	maxNode := ""
+	for k, v := range m {
+		if v > maxVal {
+			maxVal = v
+			maxNode = k
+		}
+	}
+	return maxNode, maxVal
+}

go.sum

@@ -1,5 +1,6 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
@@ -32,6 +33,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -40,6 +42,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191205161847-0a08dada0ff9 h1:abxekknhS/Drh3uoQDk5Hc7BgeiyI39Crb7vhf/1j5s=