Support NodePort and LoadBalancer services

* Added a component on the host cluster which watches for NodePort and
  LoadBalancer services, and dynamically manages a host cluster service with
  matching ports
* Updated e2e tests to use single-stage dockerfile and makefile for
  reduce build times
* Added tests for loadbalancer to k3s test case

helm/kink/templates/_helpers.tpl

@@ -31,6 +31,15 @@ If release name contains chart name it will be used as a full name.
 {{- include "kink.fullname" . }}-worker
 {{- end }}
 
+{{- define "kink.lb-manager.fullname" -}}
+{{- include "kink.fullname" . }}-lb-manager
+{{- end }}
+
+{{- define "kink.load-balancer.fullname" -}}
+{{- include "kink.fullname" . }}-lb
+{{- end }}
+
+
 {{- define "kink.kubeconfig.fullname" -}}
 {{- include "kink.fullname" . }}-kubeconfig
 {{- end }}
@@ -71,6 +80,22 @@ app.kubernetes.io/component: worker
 {{- end }}
 {{- end -}}
 
+{{- define "kink.lb-manager.labels" -}}
+{{ include "kink.labels" . }}
+app.kubernetes.io/component: lb-manager
+{{- with .Values.loadBalancer.extraLabels }}
+{{ . | toYaml }}
+{{- end }}
+{{- end -}}
+
+{{- define "kink.load-balancer.labels" -}}
+{{ include "kink.labels" . }}
+app.kubernetes.io/component: load-balancer
+{{- with .Values.loadBalancer.service.labels }}
+{{ . | toYaml }}
+{{- end }}
+{{- end -}}
+
 {{- define "kink.kubeconfig.labels" -}}
 {{ include "kink.labels" . }}
 app.kubernetes.io/component: kubeconfig
@@ -105,6 +130,15 @@ app.kubernetes.io/component: worker
 {{- end }}
 {{- end -}}
 
+{{- define "kink.lb-manager.selectorLabels" -}}
+{{ include "kink.selectorLabels" . }}
+app.kubernetes.io/component: lb-manager
+{{- with .Values.loadBalancer.extraLabels }}
+{{ . | toYaml }}
+{{- end }}
+{{- end -}}
+
+
 {{/*
 Create the name of the service account to use
 */}}
@@ -125,6 +159,14 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{- define "kink.lb-manager.serviceAccountName" -}}
+{{- if .Values.loadBalancer.manager.serviceAccount.create }}
+{{- default (include "kink.lb-manager.fullname" .) .Values.loadBalancer.manager.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.loadBalancer.manager.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
 {{- define "kink.kubeconfig.serviceAccountName" -}}
 {{- if .Values.kubeconfig.job.serviceAccount.create }}
 {{- default (include "kink.kubeconfig.fullname" .) .Values.kubeconfig.job.serviceAccount.name }}

helm/kink/templates/configmap.yaml

@@ -2,12 +2,16 @@
 fullname: {{ include "kink.fullname" . }}
 controlplane.fullname: {{ include "kink.controlplane.fullname" . }}
 controlplane.port: '{{ .Values.controlplane.service.api.port }}'
+load-balancer.fullname: {{ include "kink.load-balancer.fullname" . }}
+lb-manager.fullname: {{ include "kink.lb-manager.fullname" . }}
 labels: '{{ include "kink.labels" . | fromYaml | toJson }}'
 selectorLabels: '{{ include "kink.selectorLabels" . | fromYaml | toJson }}'
 controlplane.labels: '{{ include "kink.controlplane.labels" . | fromYaml | toJson }}'
 controlplane.selectorLabels: '{{ include "kink.controlplane.selectorLabels" . | fromYaml | toJson }}'
 worker.labels: '{{ include "kink.worker.labels" . | fromYaml | toJson }}'
 worker.selectorLabels: '{{ include "kink.worker.selectorLabels" . | fromYaml | toJson }}'
+load-balancer.labels: '{{ include "kink.load-balancer.labels" . | fromYaml | toJson }}'
+load-balancer.annotations: '{{ .Values.loadBalancer.service.annotations | toJson }}'
 rke2.enabled: '{{ .Values.rke2.enabled }}'
 {{- end -}}
 

helm/kink/templates/load-balancer/manager-deploy.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.loadBalancer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "kink.lb-manager.fullname" . }}
+  labels:
+    {{- include "kink.load-balancer.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.loadBalancer.manager.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "kink.lb-manager.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.loadBalancer.manager.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kink.lb-manager.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "kink.lb-manager.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.loadBalancer.manager.podSecurityContext | nindent 8 }}
+      containers:
+        - name: init
+          securityContext:
+            {{- toYaml .Values.loadBalancer.manager.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          {{- with .Values.extraEnv }}
+          {{- . | toYaml | nindent 10 }}
+          {{- end }}
+          {{- with .Values.loadBalancer.manager.extraEnv }}
+          {{- . | toYaml | nindent 10 }}
+          {{- end }}
+          command:
+          - kink
+          - lb-manager
+          args:
+          - --release-config-mount=/etc/kink/release 
+          - --namespace={{ .Release.Namespace }}
+          - --leader-election-id=$(POD_NAME)
+          - --guest-kubeconfig=/etc/kink/kubeconfig
+          resources:
+            {{- toYaml .Values.loadBalancer.manager.resources | nindent 12 }}
+          volumeMounts:
+          - name: release
+            mountPath: /etc/kink/release
+          - name: kubeconfig
+            mountPath: /etc/kink/kubeconfig
+            subPath: config
+          {{- with .Values.extraVolumeMounts }}
+          {{- . | toYaml | nindent 10 }}
+          {{- end }}
+          {{- with .Values.loadBalancer.manager.extraVolumeMounts }}
+          {{- . | toYaml | nindent 10 }}
+          {{- end }}
+
+      {{- with .Values.loadBalancer.manager.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.loadBalancer.manager.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.loadBalancer.manager.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+      - name: release
+        configMap:
+          name: {{ include "kink.fullname" . }} 
+      - name: kubeconfig
+        secret:
+          secretName: {{ include "kink.kubeconfig.fullname" . }}
+      {{- with .Values.extraVolumes }}
+      {{- . | toYaml | nindent 6 }}
+      {{- end }}
+      {{- with .Values.worker.extraVolumes }}
+      {{- . | toYaml | nindent 6 }}
+      {{- end }}
+{{- end }}            

helm/kink/templates/load-balancer/rbac.yaml

@@ -0,0 +1,39 @@
+{{- if and .Values.loadBalancer.enabled .Values.loadBalancer.manager.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "kink.lb-manager.fullname" . }}
+  labels:
+    {{ include "kink.lb-manager.labels" . | nindent 4 }}
+rules:
+- apiGroups: ['']
+  resources: ['services']
+  verbs: [get,watch,update,patch,delete]
+  resourceNames: ['{{ include "kink.load-balancer.fullname" . }}']
+- apiGroups: ['']
+  resources: ['services']
+  verbs: [create]
+- apiGroups: [coordination.k8s.io]
+  resources: ['leases']
+  verbs: ['*']
+  resourceNames: ['{{ include "kink.lb-manager.fullname" . }}']
+- apiGroups: [coordination.k8s.io]
+  resources: ['leases']
+  verbs: ['create']
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "kink.lb-manager.fullname" . }}
+  labels:
+    {{ include "kink.lb-manager.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kink.lb-manager.fullname" . }}
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: {{ include "kink.lb-manager.serviceAccountName" . }}
+  namespace: {{ .Release.Namespce }}
+{{- end }}

helm/kink/templates/load-balancer/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if and .Values.loadBalancer.enabled .Values.loadBalancer.manager.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "kink.lb-manager.serviceAccountName" . }}
+  labels:
+    {{- include "kink.load-balancer.labels" . | nindent 4 }}
+  {{- with .Values.loadBalancer.manager.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

helm/kink/values.yaml

@@ -202,6 +202,65 @@ worker:
   extraVolumes: []
   extraVolumeMounts: []
 
+# If enabled, an additional deployment will be created which watches the guest cluster
+#   for NodePort and LoadBalancer type services, and dynamically manages a service on
+#   the host cluster named {{ fullname }}-lb with the same ports
+loadBalancer:
+  enabled: false
+  labels: []
+  service:
+    annotations: {}
+  manager:
+    replicaCount: 1
+    serviceAccount:
+      # Specifies whether a service account should be created
+      create: true
+      # Annotations to add to the service account
+      annotations: {}
+      # The name of the service account to use.
+      # If not set and create is true, a name is generated using the fullname template
+      name: ""
+    rbac:
+      # If true, create a role and rolebinding to provide access to the dynamic service
+      create: true
+
+    extraLabels: {}
+
+    podAnnotations: {}
+
+    podSecurityContext: {}
+      # fsGroup: 2000
+
+    securityContext: {}
+      # capabilities:
+      #   drop:
+      #   - ALL
+      # readOnlyRootFilesystem: true
+      # runAsNonRoot: true
+      # runAsUser: 1000
+
+    resources: {}
+      # We usually recommend not to specify default resources and to leave this as a conscious
+      # choice for the user. This also increases chances charts run on environments with little
+      # resources, such as Minikube. If you do want to specify resources, uncomment the following
+      # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+      # limits:
+      #   cpu: 100m
+      #   memory: 128Mi
+      # requests:
+      #   cpu: 100m
+      #   memory: 128Mi
+
+    nodeSelector: {}
+
+    tolerations: []
+
+    affinity: {}
+
+    extraEnv: []
+    extraArgs: []
+    extraVolumes: []
+    extraVolumeMounts: []
 
 # If enabled, run a hook job to create a secret containing a kubeconfig usable by mounting within another pod
 kubeconfig:

integration-test/kink.k3s-ha.config.yaml

@@ -1,5 +1,8 @@
 apiVersion: kink.meln5674.github.com/v0
-Kind: Config
+kind: Config
+kubernetes:
+  context:
+    namespace: k3s-ha
 release:
   set:
     image.pullPolicy: Never
@@ -11,3 +14,4 @@ release:
     worker.persistence.enabled: "true"
     sharedPersistence.enabled: "true"
     sharedPersistence.storageClassName: "shared-local-path"
+    controlplane.extraArgs[0]: '--disable=traefik'

integration-test/kink.k3s-single.config.yaml

@@ -1,5 +1,8 @@
 apiVersion: kink.meln5674.github.com/v0
-Kind: Config
+kind: Config
+kubernetes:
+  context:
+    namespace: k3s-single
 release:
   set:
     image.pullPolicy: Never
@@ -10,4 +13,4 @@ release:
     worker.replicaCount: "0"
     sharedPersistence.enabled: "true"
     sharedPersistence.storageClassName: "shared-local-path"
-
+    controlplane.extraArgs[0]: '--disable=traefik'

integration-test/kink.k3s.config.yaml

@@ -1,5 +1,8 @@
 apiVersion: kink.meln5674.github.com/v0
-Kind: Config
+kind: Config
+kubernetes:
+  context:
+    namespace: k3s
 release:
   set:
     image.pullPolicy: Never
@@ -31,6 +34,11 @@ release:
     workers.extraVolumeMounts[0].mountPath: /tmp/baz
     extraArgs[0]: -v
     controlplane.extraArgs[0]: '--debug'
+    controlplane.extraArgs[1]: '--disable=traefik'
+    controlplane.extraArgs[2]: '--disable=servicelb'
+    controlplane.extraArgs[3]: '--service-node-port-range=30000-30001'
     workers.extraArgs[0]: '--node-label=foo=bar'
+    loadBalancer.enabled: 'true'
+    kubeconfig.enabled: 'true'
   setString:
     extraArgs[1]: '1'

integration-test/kink.rke2.config.yaml

@@ -1,5 +1,8 @@
 apiVersion: kink.meln5674.github.com/v0
-Kind: Config
+kind: Config
+kubernetes:
+  context:
+    namespace: rke2
 release:
   upgradeFlags: ['--timeout=15m']
   set:
@@ -13,3 +16,5 @@ release:
     rke2.enabled: "true"
     sharedPersistence.enabled: "true"
     sharedPersistence.storageClassName: "shared-local-path"
+    controlplane.extraArgs[0]: '--disable=rke2-ingress-nginx'
+    controlplane.extraArgs[1]: '--disable=rke2-metrics-server'

integration-test/run-case.sh

@@ -22,6 +22,7 @@ fi
 
 if ! ("${KINK_COMMAND[@]}" get cluster | tee /dev/stderr | grep -w "${KINK_CLUSTER_NAME}") || [ -z "${KINK_IT_NO_KINK_CREATE}" ]; then
     "${KINK_COMMAND[@]}" create cluster \
+        --chart ./helm/kink \
         --set image.repository="${IMAGE_REPO}" \
         --set image.tag="${IMAGE_TAG}" \
         --out-kubeconfig="${KINK_KUBECONFIG}"

pkg/config/config.go

@@ -169,12 +169,16 @@ type ReleaseConfig struct {
 	Fullname                   string    `json:"fullname"`
 	ControlplaneFullname       string    `json:"controlplane.fullname"`
 	ControlplanePort           Int       `json:"controlplane.port"`
+	LoadBalancerFullname       string    `json:"load-balancer.fullname"`
+	LBManagerFullname          string    `json:"lb-manager.fullname"`
 	Labels                     StringMap `json:"labels"`
 	SelectorLabels             StringMap `json:"selectorLabels"`
 	ControlplaneLabels         StringMap `json:"controlplane.labels"`
 	ControlplaneSelectorLabels StringMap `json:"controlplane.selectorLabels"`
 	WorkerLabels               StringMap `json:"worker.labels"`
 	WorkerSelectorLabels       StringMap `json:"worker.selectorLabels"`
+	LoadBalancerLabels         StringMap `json:"load-balancer.labels"`
+	LoadBalancerAnnotations    StringMap `json:"load-balancer.annotations"`
 	RKE2Enabled                Bool      `json:"rke2.enabled"`
 }