@@ -20,20 +20,23 @@ import (
2020 "context"
2121 "encoding/json"
2222 "fmt"
23+ "slices"
24+ "strings"
25+
2326 v1alpha1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1"
2427 "github.com/red-hat-storage/ocs-client-operator/pkg/templates"
2528 "github.com/red-hat-storage/ocs-client-operator/pkg/utils"
26- "slices"
27- "strings"
2829
2930 csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
3031 "github.com/go-logr/logr"
3132
3233 replicationv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
34+ groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
3335 snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
3436 providerclient "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/client"
3537 corev1 "k8s.io/api/core/v1"
3638 storagev1 "k8s.io/api/storage/v1"
39+ extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
3740 "k8s.io/apimachinery/pkg/api/errors"
3841 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3942 "k8s.io/apimachinery/pkg/runtime"
@@ -42,6 +45,7 @@ import (
4245 "sigs.k8s.io/controller-runtime/pkg/cache"
4346 "sigs.k8s.io/controller-runtime/pkg/client"
4447 "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
48+ "sigs.k8s.io/controller-runtime/pkg/handler"
4549 ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
4650 "sigs.k8s.io/controller-runtime/pkg/predicate"
4751 "sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -52,8 +56,10 @@ const (
5256 storageClaimAnnotation = "ocs.openshift.io/storageclaim"
5357 keyRotationAnnotation = "keyrotation.csiaddons.openshift.io/schedule"
5458
55- pvClusterIDIndexName = "index:persistentVolumeClusterID"
56- vscClusterIDIndexName = "index:volumeSnapshotContentCSIDriver"
59+ pvClusterIDIndexName = "index:persistentVolumeClusterID"
60+ vscClusterIDIndexName = "index:volumeSnapshotContentCSIDriver"
61+ vgscClusterIDIndexName = "index:volumeGroupSnapshotContentCSIDriver"
62+ volumeGroupSnapshotClassCrd = "groupsnapshot.storage.k8s.io/volumegroupsnapshotclass"
5763)
5864
5965// StorageClaimReconciler reconciles a StorageClaim object
@@ -62,6 +68,7 @@ type StorageClaimReconciler struct {
6268 cache.Cache
6369 Scheme * runtime.Scheme
6470 OperatorNamespace string
71+ AvailableCrds map [string ]bool
6572
6673 log logr.Logger
6774 ctx context.Context
@@ -104,13 +111,46 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
104111 return fmt .Errorf ("unable to set up FieldIndexer for VSC csi driver name: %v" , err )
105112 }
106113
114+ if err := mgr .GetCache ().IndexField (ctx , & groupsnapapi.VolumeGroupSnapshotContent {}, vgscClusterIDIndexName , func (o client.Object ) []string {
115+ vgsc := o .(* groupsnapapi.VolumeGroupSnapshotContent )
116+ if vgsc != nil &&
117+ slices .Contains (csiDrivers , vgsc .Spec .Driver ) &&
118+ vgsc .Status != nil &&
119+ vgsc .Status .VolumeGroupSnapshotHandle != nil {
120+ parts := strings .Split (* vgsc .Status .VolumeGroupSnapshotHandle , "-" )
121+ if len (parts ) == 9 {
122+ // second entry in the volumeID is clusterID which is unique across the cluster
123+ return []string {parts [2 ]}
124+ }
125+ }
126+ return nil
127+ }); err != nil {
128+ return fmt .Errorf ("unable to set up FieldIndexer for VSC csi driver name: %v" , err )
129+ }
130+
107131 generationChangePredicate := predicate.GenerationChangedPredicate {}
108132 bldr := ctrl .NewControllerManagedBy (mgr ).
109133 For (& v1alpha1.StorageClaim {}, builder .WithPredicates (generationChangePredicate )).
110134 Owns (& storagev1.StorageClass {}).
111135 Owns (& snapapi.VolumeSnapshotClass {}).
112- Owns (& replicationv1alpha1.VolumeReplicationClass {}, builder .WithPredicates (generationChangePredicate )).
113- Owns (& csiopv1a1.ClientProfile {}, builder .WithPredicates (generationChangePredicate ))
136+ Owns (& csiopv1a1.ClientProfile {}, builder .WithPredicates (generationChangePredicate )).
137+ Watches (
138+ & extv1.CustomResourceDefinition {},
139+ & handler.EnqueueRequestForObject {},
140+ builder .WithPredicates (
141+ utils .NamePredicate (volumeGroupSnapshotClassCrd ),
142+ utils .EventTypePredicate (
143+ ! r .AvailableCrds [volumeGroupSnapshotClassCrd ],
144+ false ,
145+ r .AvailableCrds [volumeGroupSnapshotClassCrd ],
146+ false ,
147+ ),
148+ ),
149+ builder .OnlyMetadata ,
150+ )
151+ if r .AvailableCrds [volumeGroupSnapshotClassCrd ] {
152+ bldr = bldr .Owns (& groupsnapapi.VolumeGroupSnapshotClass {})
153+ }
114154
115155 return bldr .Complete (r )
116156}
@@ -121,8 +161,10 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
121161//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete
122162//+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch;create;update;patch;delete
123163//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotclasses,verbs=get;list;watch;create;delete
164+ //+kubebuilder:rbac:groups=groupsnapshot.storage.k8s.io,resources=volumegroupsnapshotclasses,verbs=create;delete;get;list;watch
124165//+kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch
125166//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch
167+ //+kubebuilder:rbac:groups=groupsnapshot.storage.k8s.io,resources=volumegroupsnapshotcontents,verbs=get;list;watch
126168//+kubebuilder:rbac:groups=csi.ceph.io,resources=clientprofiles,verbs=get;list;update;create;watch;delete
127169//+kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplicationclasses,verbs=get;list;watch;create;delete
128170
@@ -141,6 +183,17 @@ func (r *StorageClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request
141183 r .ctx = ctrllog .IntoContext (ctx , r .log )
142184 r .log .Info ("Reconciling StorageClaim." )
143185
186+ crd := & metav1.PartialObjectMetadata {}
187+ for _ , crdName := range []string {volumeGroupSnapshotClassCrd } {
188+ crd .SetGroupVersionKind (extv1 .SchemeGroupVersion .WithKind ("CustomResourceDefinition" ))
189+ crd .Name = crdName
190+ if err := r .Client .Get (ctx , client .ObjectKeyFromObject (crd ), crd ); client .IgnoreNotFound (err ) != nil {
191+ r .log .Error (err , "Failed to get CRD" , "CRD" , crdName )
192+ return reconcile.Result {}, err
193+ }
194+ }
195+ utils .AssertEqual (r .AvailableCrds [crd .Name ], crd .UID != "" , utils .ExitCodeThatShouldRestartTheProcess )
196+
144197 // Fetch the StorageClaim instance
145198 r .storageClaim = & v1alpha1.StorageClaim {}
146199 r .storageClaim .Name = req .Name
@@ -401,7 +454,35 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) {
401454 return nil
402455 })
403456 if err != nil {
404- return reconcile.Result {}, fmt .Errorf ("failed to create or update VolumeSnapshotClass: %s" , err )
457+ return reconcile.Result {}, fmt .Errorf ("failed to create or update StorageClass: %s" , err )
458+ }
459+ case "VolumeGroupSnapshotClass" :
460+ // check for CRD availability
461+ if r .AvailableCrds [("VolumeGroupSnapshotClass" )] {
462+ var volumeGroupSnapshotClass * groupsnapapi.VolumeGroupSnapshotClass
463+ data := map [string ]string {}
464+ err = json .Unmarshal (resource .Data , & data )
465+ if err != nil {
466+ return reconcile.Result {}, fmt .Errorf ("failed to unmarshal StorageClaim configuration response: %v" , err )
467+ }
468+ data ["csi.storage.k8s.io/group-snapshotter-secret-namespace" ] = r .OperatorNamespace
469+ // generate a new clusterID for cephfs subvolumegroup, as
470+ // storageclaim is clusterscoped resources using its
471+ // hash as the clusterID
472+ data ["clusterID" ] = r .storageClaimHash
473+ driverName := templates .CephFsDriverName
474+ if strings .Contains (strings .ToLower (resource .Name ), "rbd" ) {
475+ driverName = templates .RBDDriverName
476+ }
477+ volumeGroupSnapshotClass = r .getCephDriverVolumeGroupSnapshotClass (resource .Labels , resource .Annotations , driverName )
478+ utils .AddAnnotation (volumeGroupSnapshotClass , storageClaimAnnotation , r .storageClaim .Name )
479+ err = utils .CreateOrReplace (r .ctx , r .Client , volumeGroupSnapshotClass , func () error {
480+ volumeGroupSnapshotClass .Parameters = data
481+ return nil
482+ })
483+ if err != nil {
484+ return reconcile.Result {}, fmt .Errorf ("failed to create or update VolumeGroupSnapshotClass: %s" , err )
485+ }
405486 }
406487 case "VolumeReplicationClass" :
407488 vrc := & replicationv1alpha1.VolumeReplicationClass {}
@@ -466,6 +547,11 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) {
466547 } else if exist {
467548 return reconcile.Result {}, fmt .Errorf ("one or more volumesnapshotcontents exist that are dependent on storageclaim %s" , r .storageClaim .Name )
468549 }
550+ if exist , err := r .hasVolumeGroupSnapshotContents (); err != nil {
551+ return reconcile.Result {}, fmt .Errorf ("failed to verify volumegroupsnapshotcontents dependent on storageclaim %q: %v" , r .storageClaim .Name , err )
552+ } else if exist {
553+ return reconcile.Result {}, fmt .Errorf ("one or more volumegroupsnapshotcontents exist that are dependent on storageclaim %s" , r .storageClaim .Name )
554+ }
469555
470556 // Call `RevokeStorageClaim` service on the provider server with StorageClaim as a request message.
471557 // Check if StorageClaim is still exists (it might have been manually removed during the StorageClass
@@ -550,6 +636,20 @@ func (r *StorageClaimReconciler) getCephRBDVolumeSnapshotClass() *snapapi.Volume
550636 return volumesnapshotclass
551637}
552638
639+ func (r * StorageClaimReconciler ) getCephDriverVolumeGroupSnapshotClass (
640+ labels map [string ]string , annotations map [string ]string , driver string ) * groupsnapapi.VolumeGroupSnapshotClass {
641+ volumegroupsnapshotclass := & groupsnapapi.VolumeGroupSnapshotClass {
642+ ObjectMeta : metav1.ObjectMeta {
643+ Name : r .storageClaim .Name ,
644+ Labels : labels ,
645+ Annotations : annotations ,
646+ },
647+ Driver : driver ,
648+ DeletionPolicy : snapapi .VolumeSnapshotContentDelete ,
649+ }
650+ return volumegroupsnapshotclass
651+ }
652+
553653func (r * StorageClaimReconciler ) get (obj client.Object ) error {
554654 key := client .ObjectKeyFromObject (obj )
555655 return r .Client .Get (r .ctx , key , obj )
@@ -594,3 +694,17 @@ func (r *StorageClaimReconciler) hasVolumeSnapshotContents() (bool, error) {
594694
595695 return false , nil
596696}
697+
698+ func (r * StorageClaimReconciler ) hasVolumeGroupSnapshotContents () (bool , error ) {
699+ vscList := & groupsnapapi.VolumeGroupSnapshotContentList {}
700+ if err := r .list (vscList , client.MatchingFields {vgscClusterIDIndexName : r .storageClaimHash }); err != nil {
701+ return false , fmt .Errorf ("failed to list volume group snapshot content resources: %v" , err )
702+ }
703+
704+ if len (vscList .Items ) != 0 {
705+ r .log .Info (fmt .Sprintf ("VolumeGroupSnapshotContent referring storageclaim %q exists" , r .storageClaim .Name ))
706+ return true , nil
707+ }
708+
709+ return false , nil
710+ }
0 commit comments