Merge pull request #546 from red-hat-storage/sync_ds--release-1.13

Syncing latest changes from release-1.13 for rook

Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md

@@ -20,8 +20,18 @@ metadata:
   name: group-a
   namespace: rook-ceph # namespace:cluster
 spec:
+  # The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
+  name: csi
   # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
   filesystemName: myfs
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)
 ```
 
 ## Settings
@@ -34,4 +44,15 @@ If any setting is unspecified, a suitable default will be used automatically.
 
 ### CephFilesystemSubVolumeGroup spec
 
+* `name`: The spec name that will be used for the Ceph Filesystem subvolume group if not set metadata name will be used.
+
 * `filesystemName`: The metadata name of the CephFilesystem CR where the subvolume group will be created.
+
+* `pinning`: To distribute load across MDS ranks in predictable and stable ways. Reference: https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups.
+    * `distributed`: Range: <0, 1>, for disabling it set to 0
+    * `export`: Range: <0-256>, for disabling it set to -1
+    * `random`: Range: [0.0, 1.0], for disabling it set to 0.0
+
+!!! note
+    Only one out of (export, distributed, random) can be set at a time.
+    By default pinning is set with value: `distributed=1`.

Documentation/CRDs/specification.md

@@ -1546,6 +1546,22 @@ list of Ceph Filesystem volumes with <code>ceph fs volume ls</code>. To learn mo
 abstractions see <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes</a></p>
 </td>
 </tr>
+<tr>
+<td>
+<code>pinning</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">
+CephFilesystemSubVolumeGroupSpecPinning
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Pinning configuration of CephFilesystemSubVolumeGroup,
+reference <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups</a>
+only one out of (export, distributed, random) can be set at a time</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -3628,6 +3644,73 @@ list of Ceph Filesystem volumes with <code>ceph fs volume ls</code>. To learn mo
 abstractions see <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes</a></p>
 </td>
 </tr>
+<tr>
+<td>
+<code>pinning</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">
+CephFilesystemSubVolumeGroupSpecPinning
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Pinning configuration of CephFilesystemSubVolumeGroup,
+reference <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups</a>
+only one out of (export, distributed, random) can be set at a time</p>
+</td>
+</tr>
+</tbody>
+</table>
+<h3 id="ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">CephFilesystemSubVolumeGroupSpecPinning
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpec">CephFilesystemSubVolumeGroupSpec</a>)
+</p>
+<div>
+<p>CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>export</code><br/>
+<em>
+int
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
+<tr>
+<td>
+<code>distributed</code><br/>
+<em>
+int
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
+<tr>
+<td>
+<code>random,</code><br/>
+<em>
+float64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.CephFilesystemSubVolumeGroupStatus">CephFilesystemSubVolumeGroupStatus

deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml

@@ -2,6 +2,25 @@
 {{- range $filesystem := .Values.cephFileSystems -}}
 ---
 apiVersion: ceph.rook.io/v1
+kind: CephFilesystemSubVolumeGroup
+metadata:
+  name: {{ $filesystem.name }}-csi    # lets keep the svg crd name same as `filesystem name + csi` for the default csi svg
+  namespace: {{ $root.Release.Namespace }} # namespace:cluster
+spec:
+  # The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
+  name: csi
+  # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
+  filesystemName: {{ $filesystem.name }}
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)
+---
+apiVersion: ceph.rook.io/v1
 kind: CephFilesystem
 metadata:
   name: {{ $filesystem.name }}

deploy/charts/rook-ceph/templates/resources.yaml

@@ -36,9 +36,15 @@ spec:
                 blockPoolName:
                   description: BlockPoolName is the name of Ceph BlockPool. Typically it's the name of the CephBlockPool CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: blockPoolName is immutable
+                      rule: self == oldSelf
                 name:
                   description: The name of the CephBlockPoolRadosNamespaceSpec namespace. If not set, the default is the name of the CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: name is immutable
+                      rule: self == oldSelf
               required:
                 - blockPoolName
               type: object
@@ -7987,9 +7993,37 @@ spec:
                 filesystemName:
                   description: FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of the CephFilesystem CR. If not coming from the CephFilesystem CR, it can be retrieved from the list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes
                   type: string
+                  x-kubernetes-validations:
+                    - message: filesystemName is immutable
+                      rule: self == oldSelf
                 name:
                   description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: name is immutable
+                      rule: self == oldSelf
+                pinning:
+                  description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time
+                  properties:
+                    distributed:
+                      maximum: 1
+                      minimum: 0
+                      nullable: true
+                      type: integer
+                    export:
+                      maximum: 256
+                      minimum: -1
+                      nullable: true
+                      type: integer
+                    random:
+                      maximum: 1
+                      minimum: 0
+                      nullable: true
+                      type: number
+                  type: object
+                  x-kubernetes-validations:
+                    - message: only one pinning type should be set
+                      rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))
               required:
                 - filesystemName
               type: object

deploy/examples/crds.yaml

@@ -39,9 +39,15 @@ spec:
                 blockPoolName:
                   description: BlockPoolName is the name of Ceph BlockPool. Typically it's the name of the CephBlockPool CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: blockPoolName is immutable
+                      rule: self == oldSelf
                 name:
                   description: The name of the CephBlockPoolRadosNamespaceSpec namespace. If not set, the default is the name of the CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: name is immutable
+                      rule: self == oldSelf
               required:
                 - blockPoolName
               type: object
@@ -7981,9 +7987,37 @@ spec:
                 filesystemName:
                   description: FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of the CephFilesystem CR. If not coming from the CephFilesystem CR, it can be retrieved from the list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes
                   type: string
+                  x-kubernetes-validations:
+                    - message: filesystemName is immutable
+                      rule: self == oldSelf
                 name:
                   description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
                   type: string
+                  x-kubernetes-validations:
+                    - message: name is immutable
+                      rule: self == oldSelf
+                pinning:
+                  description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time
+                  properties:
+                    distributed:
+                      maximum: 1
+                      minimum: 0
+                      nullable: true
+                      type: integer
+                    export:
+                      maximum: 256
+                      minimum: -1
+                      nullable: true
+                      type: integer
+                    random:
+                      maximum: 1
+                      minimum: 0
+                      nullable: true
+                      type: number
+                  type: object
+                  x-kubernetes-validations:
+                    - message: only one pinning type should be set
+                      rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))
               required:
                 - filesystemName
               type: object

deploy/examples/filesystem-ec.yaml

@@ -91,3 +91,23 @@ spec:
     #    cpu: "500m"
     #    memory: "1024Mi"
     # priorityClassName: my-priority-class
+---
+# create default csi subvolume group
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystemSubVolumeGroup
+metadata:
+  name: myfs-csi # lets keep the svg crd name same as `filesystem name + csi` for the default csi svg
+  namespace: rook-ceph # namespace:cluster
+spec:
+  # The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
+  name: csi
+  # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
+  filesystemName: myfs
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)

deploy/examples/filesystem-test.yaml

@@ -23,3 +23,23 @@ spec:
   metadataServer:
     activeCount: 1
     activeStandby: true
+---
+# create default csi subvolume group
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystemSubVolumeGroup
+metadata:
+  name: myfs-csi # lets keep the svg crd name same as `filesystem name + csi` for the default csi svg
+  namespace: rook-ceph # namespace:cluster
+spec:
+  # The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
+  name: csi
+  # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
+  filesystemName: myfs
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)

deploy/examples/filesystem.yaml

@@ -136,3 +136,23 @@ spec:
     # snapshotRetention:
     #   - path: /
     #     duration: "h 24"
+---
+# create default csi subvolume group
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystemSubVolumeGroup
+metadata:
+  name: myfs-csi # lets keep the svg crd name same as `filesystem name + csi` for the default csi svg
+  namespace: rook-ceph # namespace:cluster
+spec:
+  # The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
+  name: csi
+  # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
+  filesystemName: myfs
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)

deploy/examples/subvolumegroup.yaml

@@ -9,3 +9,11 @@ spec:
   # name: group-a
   # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
   filesystemName: myfs
+  # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
+  # only one out of (export, distributed, random) can be set at a time
+  # by default pinning is set with value: distributed=1
+  # for disabling default values set (distributed=0)
+  pinning:
+    distributed: 1            # distributed=<0, 1> (disabled=0)
+    # export:                 # export=<0-256> (disabled=-1)
+    # random:                 # random=[0.0, 1.0](disabled=0.0)