[WIP] Upgrade Ceph CSI CephFS provisioner 3.11.0

README.md

@@ -185,7 +185,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [metallb](https://metallb.universe.tf/)  v0.13.9
   - [registry](https://github.com/distribution/distribution) v2.8.1
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v3.11.0
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml

@@ -1,10 +1,24 @@
 ---
 cephfs_provisioner_namespace: "cephfs-provisioner"
-cephfs_provisioner_cluster: ceph
-cephfs_provisioner_monitors: ~
-cephfs_provisioner_admin_id: admin
-cephfs_provisioner_secret: secret
-cephfs_provisioner_storage_class: cephfs
+cephfs_provisioner_csidriver_name: cephfs.csi.ceph.com
+cephfs_provisioner_secret_name: csi-cephfs-secret
+cephfs_provisioner_cluster_id: ""
+cephfs_provisioner_admin_id: ""
+cephfs_provisioner_admin_key: ""
+cephfs_provisioner_storage_class: csi-cephfs-sc
+cephfs_provisioner_allow_volume_expansion: true
 cephfs_provisioner_reclaim_policy: Delete
-cephfs_provisioner_claim_root: /volumes
-cephfs_provisioner_deterministic_names: true
+cephfs_provisioner_mount_options: []
+cephfs_provisioner_ceph_config: |
+  [global]
+  auth_cluster_required = cephx
+  auth_service_required = cephx
+  auth_client_required = cephx
+
+  # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
+  # adding 'fuse_big_writes = true' option by default to override this limit
+  # see https://github.com/ceph/ceph-csi/issues/1928
+  fuse_big_writes = true
+
+cephfs_provisioner_ceph_csi_config_json: |
+  []

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml

@@ -41,14 +41,23 @@
   set_fact:
     cephfs_provisioner_templates:
       - { name: 00-namespace, file: 00-namespace.yml, type: ns }
-      - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret }
+      - { name: sa-cephfs-csi-cephfs-nodeplugin, file: sa-cephfs-csi-cephfs-nodeplugin.yml, type: sa }
       - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa }
-      - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole }
-      - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding }
-      - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role }
-      - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding }
-      - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy }
-      - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc }
+      - { name: secret-csi-cephfs-secret, file: secret-csi-cephfs-secret.yml, type: secret }
+      - { name: cm-ceph-config, file: cm-ceph-config.yml, type: cm }
+      - { name: cm-ceph-csi-config, file: cm-ceph-csi-config.yml, type: cm }
+      - { name: sc-csi-cephfs-sc, file: sc-csi-cephfs-sc.yml, type: storageclass }
+      - { name: clusterrole-ceph-csi-cephfs-nodeplugin, file: clusterrole-ceph-csi-cephfs-nodeplugin.yml, type: clusterrole }
+      - { name: clusterrole-ceph-csi-cephfs-provisioner, file: clusterrole-ceph-csi-cephfs-provisioner.yml, type: clusterrole }
+      - { name: clusterrolebinding-ceph-csi-cephfs-nodeplugin, file: clusterrolebinding-ceph-csi-cephfs-nodeplugin.yml, type: clusterrolebinding }
+      - { name: clusterrolebinding-ceph-csi-cephfs-provisioner, file: clusterrolebinding-ceph-csi-cephfs-provisioner.yml, type: clusterrolebinding }
+      - { name: role-ceph-csi-cephfs-provisioner, file: role-ceph-csi-cephfs-provisioner.yml, type: role }
+      - { name: rolebinding-ceph-csi-cephfs-provisioner, file: rolebinding-ceph-csi-cephfs-provisioner.yml, type: rolebinding }
+      - { name: svc-ceph-csi-cephfs-nodeplugin-http-metrics, file: svc-ceph-csi-cephfs-nodeplugin-http-metrics.yml, type: service }
+      - { name: svc-ceph-csi-cephfs-provisioner-http-metrics, file: svc-ceph-csi-cephfs-provisioner-http-metrics.yml, type: service}
+      - { name: daemonset-csi-cephfsplugin, file: daemonset-csi-cephfsplugin.yml, type: daemonset }
+      - { name: deploy-csi-cephfsplugin-provisioner, file: deploy-csi-cephfsplugin-provisioner.yml, type: deploy }
+      - { name: csidriver-cephfs, file: csidriver-cephfs.yml, type: CSIDriver }
 
 - name: CephFS Provisioner | Create manifests
   template:

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-ceph-csi-cephfs-nodeplugin.yml.j2

@@ -0,0 +1,15 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ceph-csi-cephfs-nodeplugin
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-ceph-csi-cephfs-provisioner.yml.j2

@@ -0,0 +1,42 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ceph-csi-cephfs-provisioner
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete","patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 → roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-ceph-csi-cephfs-nodeplugin.yml.j2

@@ -1,13 +1,13 @@
 ---
-apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: cephfs-provisioner
+  name: ceph-csi-cephfs-nodeplugin
 subjects:
   - kind: ServiceAccount
-    name: cephfs-provisioner
+    name: ceph-csi-cephfs-nodeplugin
     namespace: {{ cephfs_provisioner_namespace }}
 roleRef:
   kind: ClusterRole
-  name: cephfs-provisioner
+  name: ceph-csi-cephfs-nodeplugin
   apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-ceph-csi-cephfs-provisioner.yml.j2

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ceph-csi-cephfs-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: ceph-csi-cephfs-provisioner
+    namespace: {{ cephfs_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ceph-csi-cephfs-provisioner
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cm-ceph-config.yml.j2

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "ceph-config"
+  namespace: {{ cephfs_provisioner_namespace }}
+data:
+  ceph.conf: |
+    {{ cephfs_provisioner_ceph_config | indent(width=4, first=False) }}
+
+  keyring: ""

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cm-ceph-csi-config.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "ceph-csi-config"
+  namespace: {{ cephfs_provisioner_namespace }}
+data:
+  config.json: |-
+    {{ cephfs_provisioner_ceph_csi_config_json | indent(width=4, first=False) }}

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/csidriver-cephfs.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: {{ cephfs_provisioner_csidriver_name }}
+  labels:
+    app: ceph-csi-cephfs
+spec:
+  attachRequired: false
+  podInfoOnMount: false
+  fsGroupPolicy: File
+  seLinuxMount: true