Update to rook v1.9.10, Rename cephfs storageclass

- cephfs storageclass, ceph-file-sc -> csi-cephfs-sc
- Update inventory to rook v1.9.10

Author: donggyupark

docs/examples/file-nginx.yaml

@@ -8,7 +8,7 @@ spec:
   resources:
     requests:
       storage: 100Mi
-  storageClassName: ceph-file-sc
+  storageClassName: csi-cephfs-sc
 ---
 apiVersion: apps/v1
 kind: Deployment

docs/file.md

@@ -102,7 +102,7 @@ spec:
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: ceph-file-sc
+  name: csi-cephfs-sc
 provisioner: rook-ceph.cephfs.csi.ceph.com
 parameters:
   # clusterID is the namespace where operator is deployed.

hack/inventory/production-sample/rook/block_pool.yaml

@@ -62,6 +62,3 @@ spec:
   # quotas:
     # maxSize: "10Gi" # valid suffixes include k, M, G, T, P, E, Ki, Mi, Gi, Ti, Pi, Ei
     # maxObjects: 1000000000 # 1 billion objects
-  # A key/value list of annotations
-  annotations:
-  #  key: value

hack/inventory/production-sample/rook/block_sc.yaml

@@ -1,19 +1,3 @@
-apiVersion: ceph.rook.io/v1
-kind: CephBlockPool
-metadata:
-  name: replicapool
-  namespace: rook-ceph
-spec:
-  failureDomain: host
-  replicated:
-    size: 3
-    # Disallow setting pool with replica 1, this could lead to data loss without recovery.
-    # Make sure you're *ABSOLUTELY CERTAIN* that is what you want
-    requireSafeReplicaSize: true
-    # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
-    # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
-    #targetSizeRatio: .5
----
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
@@ -47,6 +31,15 @@ parameters:
   # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
   # unmapOptions: force
 
+   # (optional) Set it to true to encrypt each volume with encryption keys
+   # from a key management system (KMS)
+   # encrypted: "true"
+
+   # (optional) Use external key management system (KMS) for encryption key by
+   # specifying a unique ID matching a KMS ConfigMap. The ID is only used for
+   # correlation to configmap entry.
+   # encryptionKMSID: <kms-config-id>
+
   # RBD image format. Defaults to "2".
   imageFormat: "2"
 

hack/inventory/production-sample/rook/cluster.yaml

@@ -16,13 +16,13 @@ metadata:
 spec:
   cephVersion:
     # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw).
-    # v14 is nautilus, v15 is octopus, and v16 is pacific.
+    # v15 is octopus, and v16 is pacific.
     # RECOMMENDATION: In production, use a specific version tag instead of the general v14 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.6-20210918
+    # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.10-20220721
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v15.2.8
-    # Whether to allow unsupported versions of Ceph. Currently `nautilus`, `octopus`, and `pacific` are supported.
+    image: quay.io/ceph/ceph:v16.2.10
+    # Whether to allow unsupported versions of Ceph. Currently `octopus` and `pacific` are supported.
     # Future versions such as `pacific` would require this to be set to `true`.
     # Do not set to true in production.
     allowUnsupported: false
@@ -33,13 +33,13 @@ spec:
   # Whether or not upgrade should continue even if a check fails
   # This means Ceph's status could be degraded and we don't recommend upgrading but you might decide otherwise
   # Use at your OWN risk
-  # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/master/ceph-upgrade.html#ceph-version-upgrades
+  # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/latest/ceph-upgrade.html#ceph-version-upgrades
   skipUpgradeChecks: false
   # Whether or not continue if PGs are not clean during an upgrade
   continueUpgradeAfterChecksEvenIfNotHealthy: false
   # WaitTimeoutForHealthyOSDInMinutes defines the time (in minutes) the operator would wait before an OSD can be stopped for upgrade or restart.
   # If the timeout exceeds and OSD is not ok to stop, then the operator would skip upgrade for the current OSD and proceed with the next one
-  # if `continueUpgradeAfterChecksEvenIfNotHealthy` is `false`. If `continueUpgradeAfterChecksEvenIfNotHealthy` is `true`, then opertor would
+  # if `continueUpgradeAfterChecksEvenIfNotHealthy` is `false`. If `continueUpgradeAfterChecksEvenIfNotHealthy` is `true`, then operator would
   # continue with the upgrade of an OSD even if its not ok to stop after the timeout. This timeout won't be applied if `skipUpgradeChecks` is `true`.
   # The default wait timeout is 10 minutes.
   waitTimeoutForHealthyOSDInMinutes: 10
@@ -54,7 +54,8 @@ spec:
     # When higher availability of the mgr is needed, increase the count to 2.
     # In that case, one mgr will be active and one in standby. When Ceph updates which
     # mgr is active, Rook will update the mgr services to match the active mgr.
-    count: 1
+    count: 2
+    allowMultiplePerNode: false
     modules:
       # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules
       # are already enabled by other settings in the cluster CR.
@@ -73,13 +74,20 @@ spec:
   monitoring:
     # requires Prometheus to be pre-installed
     enabled: false
-    # namespace to deploy prometheusRule in. If empty, namespace of the cluster will be used.
-    # Recommended:
-    # If you have a single rook-ceph cluster, set the rulesNamespace to the same namespace as the cluster or keep it empty.
-    # If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus
-    # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions.
-    rulesNamespace: rook-ceph
   network:
+    connections:
+      # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network.
+      # The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted.
+      # When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check.
+      # IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only,
+      # you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class.
+      # The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes.
+      encryption:
+        enabled: false
+      # Whether to compress the data in transit across the wire. The default is false.
+      # Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption.
+      compression:
+        enabled: false
     # enable host networking
     #provider: host
     # enable the Multus network provider
@@ -107,8 +115,9 @@ spec:
   # enable log collector, daemons will log on files and rotate
   # logCollector:
   #   enabled: true
-  #   periodicity: 24h # SUFFIX may be 'h' for hours or 'd' for days.
-  # automate [data cleanup process](https://github.com/rook/rook/blob/master/Documentation/ceph-teardown.md#delete-the-data-on-hosts) in cluster destruction.
+  #   periodicity: daily # one of: hourly, daily, weekly, monthly
+  #   maxLogSize:  500M # SUFFIX may be 'M' or 'G'. Must be at least 1M.
+  # automate [data cleanup process](https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/ceph-teardown.md#delete-the-data-on-hosts) in cluster destruction.
   cleanupPolicy:
     # Since cluster cleanup is destructive to data, confirmation is required.
     # To destroy all Rook data on hosts during uninstall, confirmation must be set to "yes-really-destroy-data".
@@ -158,6 +167,7 @@ spec:
 # or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a
 # preferred rule with weight: 50.
 #    osd:
+#    prepareosd:
 #    mgr:
 #    cleanup:
   annotations:
@@ -166,6 +176,10 @@ spec:
 #    osd:
 #    cleanup:
 #    prepareosd:
+# clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets.
+# And clusterMetadata annotations will not be merged with `all` annotations.
+#    clusterMetadata:
+#       kubed.appscode.com/sync: "true"
 # If no mgr annotations are set, prometheus scrape annotations will be set by default.
 #    mgr:
   labels:
@@ -178,22 +192,23 @@ spec:
 # monitoring is a list of key-value pairs. It is injected into all the monitoring resources created by operator.
 # These labels can be passed as LabelSelector to Prometheus
 #    monitoring:
+#    crashcollector:
   resources:
 # The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory
 #    mgr:
 #      limits:
-#        cpu: "1"
+#        cpu: "500m"
 #        memory: "1024Mi"
 #      requests:
-#        cpu: "1"
+#        cpu: "500m"
 #        memory: "1024Mi"
 # The above example requests/limits can also be added to the other components
 #    mon:
 #      limits:
-#        cpu: "2"
+#        cpu: "1"
 #        memory: "2048Mi"
 #      requests:
-#        cpu: "2"
+#        cpu: "1"
 #        memory: "2048Mi"
 #    osd:
 #      limits:
@@ -214,10 +229,11 @@ spec:
   # The option to automatically remove OSDs that are out and are safe to destroy.
   removeOSDsIfOutAndSafeToRemove: false
   priorityClassNames:
-    all: rook-ceph-default-priority-class
-#    mon: rook-ceph-mon-priority-class
-#    osd: rook-ceph-osd-priority-class
-#    mgr: rook-ceph-mgr-priority-class
+    #all: rook-ceph-default-priority-class
+    mon: system-node-critical
+    osd: system-node-critical
+    mgr: system-cluster-critical
+    #crashcollector: rook-ceph-crashcollector-priority-class
   storage: # cluster level storage configuration and selection
     useAllNodes: false
     useAllDevices: false
@@ -249,7 +265,7 @@ spec:
     # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically
     # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will
     # block eviction of OSDs by default and unblock them safely when drains are detected.
-    managePodBudgets: false
+    managePodBudgets: true
     # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the
     # default DOWN/OUT interval) when it is draining. This is only relevant when  `managePodBudgets` is `true`. The default value is `30` minutes.
     osdMaintenanceTimeout: 30
@@ -276,11 +292,19 @@ spec:
       status:
         disabled: false
         interval: 60s
-    # Change pod liveness probe, it works for all mon,mgr,osd daemons
+    # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons.
     livenessProbe:
       mon:
         disabled: false
       mgr:
         disabled: false
       osd:
         disabled: false
+    # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons.
+    startupProbe:
+      mon:
+        disabled: false
+      mgr:
+        disabled: false
+      osd:
+        disabled: false