Convert Rook NFS to Helm chart

- Adds Rook NFS Helm chart as dependency of Slurm cluster chart
- Refactors main values file to allow additional customisation
- Adds cleanup job as pre-delete hook to fix uninstall behaviour

.gitignore

@@ -0,0 +1,3 @@
+# Build artifacts from local helm install
+slurm-cluster-chart/Chart.lock
+slurm-cluster-chart/charts/

rooknfs/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v2
+name: rooknfs
+version: 0.0.1
+description: An packaged installation of Rook NFS for Kubernetes.

nfs/nfs.yaml → rooknfs/templates/nfs.yaml

@@ -1,32 +1,36 @@
+{{- if .Values.enabled }}
 ---
 # A default storageclass must be present
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: nfs-default-claim
-  namespace: rook-nfs
+  name: {{ .Values.claimName}}
+  namespace: {{ .Values.serverNamespace }}
 spec:
   accessModes:
   - ReadWriteMany
   resources:
     requests:
-      storage: 1Gi
+      storage: {{ .Values.storageCapacity }}
 ---
 apiVersion: nfs.rook.io/v1alpha1
 kind: NFSServer
 metadata:
-  name: rook-nfs
-  namespace: rook-nfs
+  name: {{ .Values.serverName }}
+  namespace: {{ .Values.serverNamespace }}
 spec:
   replicas: 1
   exports:
-  - name: share1
+  - name: {{ .Values.shareName }}
     server:
       accessMode: ReadWrite
       squash: "none"
     # A Persistent Volume Claim must be created before creating NFS CRD instance.
     persistentVolumeClaim:
-      claimName: nfs-default-claim
+      claimName: {{ .Values.claimName }}
   # A key/value list of annotations
   annotations:
     rook: nfs
+---
+{{- end }}
+

nfs/operator.yaml → rooknfs/templates/operator.yaml

@@ -1,13 +1,15 @@
+{{- if .Values.enabled }}
+---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: rook-nfs-system # namespace:operator
+  name: {{ .Values.systemNamespace }}
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: rook-nfs-operator
-  namespace: rook-nfs-system # namespace:operator
+  namespace: {{ .Values.systemNamespace }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -20,7 +22,7 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: rook-nfs-operator
-    namespace: rook-nfs-system # namespace:operator
+    namespace:  {{ .Values.systemNamespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -106,7 +108,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: rook-nfs-operator
-  namespace: rook-nfs-system # namespace:operator
+  namespace: {{ .Values.systemNamespace }}
   labels:
     app: rook-nfs-operator
 spec:
@@ -134,3 +136,5 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+---
+{{- end}}

nfs/rbac.yaml → rooknfs/templates/rbac.yaml

@@ -1,14 +1,15 @@
+{{- if .Values.enabled }}
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name:  rook-nfs
+  name:  {{ .Values.serverNamespace }}
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: rook-nfs-server
-  namespace: rook-nfs
+  namespace:  {{ .Values.serverNamespace }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -51,9 +52,10 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: rook-nfs-server
-     # replace with namespace where provisioner is deployed
-    namespace: rook-nfs
+    namespace:  {{ .Values.serverNamespace }}
 roleRef:
   kind: ClusterRole
   name: rook-nfs-provisioner-runner
   apiGroup: rbac.authorization.k8s.io
+---
+{{- end }}

rooknfs/templates/sc.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.enabled }}
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  labels:
+    app: rook-nfs
+  name: {{ .Values.storageClassName }}
+parameters:
+  exportName: {{ .Values.shareName }}
+  nfsServerName: {{ .Values.serverName }}
+  nfsServerNamespace:  {{ .Values.serverNamespace }}
+provisioner: nfs.rook.io/rook-nfs-provisioner
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+---
+{{- end }}

rooknfs/values.yaml

@@ -0,0 +1,30 @@
+# Global flag for enabling/disabling all chart resources
+# This is useful for allowing charts which use this chart
+# as a dependency to toggle usage of this chart based on
+# values in the parent chart
+enabled: true
+
+# Name for the NFSServer resource created by rook
+serverName: rook-nfs
+
+# Name for the created storage class
+storageClassName: rook-nfs
+
+# Name for the Read-Write-Once backing PVC created by Rook
+claimName: rook-nfs-backing-pv
+
+# Name for the NFS share within the NFS Resource instance
+shareName: share-1
+
+# Size of the Read-Write-Once backing storage volume
+storageCapacity: 10Gi
+
+# Image to use for the Rook NFS operator
+operatorImage: rook/nfs:master
+
+# NOTE: For some reason deploying everything in the default
+# namespace leads to R-W-M PVCs getting stuck in 'pending'
+# state indefinitely, so here we separate out namespaces as
+# of various components in the same way as the Rook docs
+serverNamespace: rook-nfs
+systemNamespace: rook-nfs-system

slurm-cluster-chart/Chart.yaml

@@ -21,4 +21,9 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.16.0"
+appVersion: "1.16.0"
+
+dependencies:
+  - name: rooknfs
+    version: 0.0.1
+    repository: file://../rooknfs

slurm-cluster-chart/templates/hooks/pre-delete.yaml

@@ -0,0 +1,55 @@
+{{- if .Values.rooknfs.enabled }}
+# NOTE: The cleanup jobs defined here are required to ensure that things which
+# Rook NFS is responsible for cleaning up are deleted before deleting the Rook
+# pods which do the actual clean up of NFS resources. For example, the RWM PVC
+# must be deleted before the Rook StorageClass and provisioner pod. However,
+# the PVC cannot be deleted until the pods which are using it are deleted, so
+# the various Slurm node pods must actually be the first resources deleted.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rook-nfs-cleanup
+---
+# TODO: Create a job-specific ClusterRole for the ServiceAccount
+# instead of using the cluster-admin role here
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-nfs-cleanup
+subjects:
+- kind: ServiceAccount
+  name: rook-nfs-cleanup
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: rook-nfs-pre-delete-cleanup
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      name: rook-nfs-pre-delete-cleanup
+    spec:
+      serviceAccountName:  rook-nfs-cleanup
+      containers:
+      - name: tester
+        image: bitnami/kubectl
+        command:
+         - "bin/bash"
+         - "-c"
+         - | 
+            kubectl delete -n {{ .Release.Namespace }} deployment {{ .Values.login.name }} --wait --cascade=foreground
+            kubectl delete -n {{ .Release.Namespace }} statefulset {{ .Values.slurmctld.name }} --wait --cascade=foreground
+            kubectl delete -n {{ .Release.Namespace }} statefulset {{ .Values.slurmd.name }} --wait --cascade=foreground
+            kubectl delete -n {{ .Release.Namespace }} pvc {{ .Values.storage.claimName }} --wait
+            kubectl delete -n {{ .Values.rooknfs.serverNamespace }} nfsservers {{ .Values.rooknfs.serverName }} --wait
+      restartPolicy: Never
+---
+{{- end }}

slurm-cluster-chart/templates/login-deployment.yaml → slurm-cluster-chart/templates/login.yaml

@@ -5,9 +5,9 @@ metadata:
   labels:
     app.kubernetes.io/name: slurm
     app.kubernetes.io/component: login
-  name: login
+  name: {{ .Values.login.name }}
 spec:
-  replicas: {{ .Values.replicas.login }}
+  replicas: {{ .Values.login.replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: slurm
@@ -29,7 +29,7 @@ spec:
           ports:
             - containerPort: 22
           volumeMounts:
-            - mountPath: {{ .Values.nfs.mountPath }}
+            - mountPath: {{ .Values.storage.mountPath }}
               name: slurm-jobdir
             - mountPath: /etc/slurm/
               name: slurm-config-volume
@@ -51,7 +51,7 @@ spec:
       volumes:
         - name: slurm-jobdir
           persistentVolumeClaim:
-            claimName: {{ .Values.nfs.claimName }}
+            claimName: {{ .Values.storage.claimName }}
         - name: slurm-config-volume
           configMap:
             name: {{ .Values.configmaps.slurmConf }}

slurm-cluster-chart/templates/pvc.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.rooknfs.enabled }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Values.storage.claimName }}
+spec:
+  storageClassName: {{ .Values.storageClassName }}
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ .Values.storage.capacity }}
+{{- end }}

slurm-cluster-chart/templates/slurmctld-statefulset.yaml → slurm-cluster-chart/templates/slurmctld.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app.kubernetes.io/name: slurm
     app.kubernetes.io/component: slurmctld
-  name: slurmctld
+  name: {{ .Values.slurmctld.name }}
 spec:
   replicas: 1
   selector:
@@ -29,7 +29,7 @@ spec:
             - containerPort: 6817
           resources: {}
           volumeMounts:
-            - mountPath: {{ .Values.nfs.mountPath }}
+            - mountPath: {{ .Values.storage.mountPath }}
               name: slurm-jobdir
             - mountPath: /etc/slurm/
               name: slurm-config-volume
@@ -45,7 +45,7 @@ spec:
       volumes:
         - name: slurm-jobdir
           persistentVolumeClaim:
-            claimName: {{ .Values.nfs.claimName }}
+            claimName: {{ .Values.storage.claimName }}
         - name: slurmctld-state
           persistentVolumeClaim:
             claimName: var-spool-slurmctld