From 71ea1e2df1b2891ed3070354da771df9a33abdad Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Mon, 15 Apr 2024 18:18:58 +0200 Subject: [PATCH] Link PVCs and PVs in VolumeGroupSnapshot and VolumeGroupSnapshotContent This use the update API to set `persistentVolumeClaimRef` in `VolumeGroupSnapshot` and `persistentVolumeName` in `VolumeGroupSnapshotContent` to the corresponding objects. This makes restoring volumes from a VolumeGroupSnapshot easier. Related: #969 --- .../groupsnapshot_controller_helper.go | 54 +++++++- .../groupsnapshot_helper.go | 54 ++++++-- pkg/utils/snapshot_info.go | 61 +++++++++ pkg/utils/snapshot_info_test.go | 129 ++++++++++++++++++ pkg/utils/util.go | 3 + 5 files changed, 289 insertions(+), 12 deletions(-) create mode 100644 pkg/utils/snapshot_info.go create mode 100644 pkg/utils/snapshot_info_test.go diff --git a/pkg/common-controller/groupsnapshot_controller_helper.go b/pkg/common-controller/groupsnapshot_controller_helper.go index c66ff4814..69da282c2 100644 --- a/pkg/common-controller/groupsnapshot_controller_helper.go +++ b/pkg/common-controller/groupsnapshot_controller_helper.go @@ -564,16 +564,39 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotStatus(groupSnapshot volumeSnapshotErr = groupSnapshotContent.Status.Error.DeepCopy() } + var snapshotInfoList utils.SnapshotInfoList + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) { + var err error + snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo]) + if err != nil { + klog.V(1).Infof( + "updateGroupSnapshotStatus[%s]: the content of the [%s] annotation is not valid: %s", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + err.Error(), + ) + } + } else { + klog.V(2).Infof( + "updateGroupSnapshotStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + ) + } + var pvcVolumeSnapshotRefList []crdv1alpha1.PVCVolumeSnapshotPair if groupSnapshotContent.Status != nil && len(groupSnapshotContent.Status.PVVolumeSnapshotContentList) != 0 { for _, contentRef := range groupSnapshotContent.Status.PVVolumeSnapshotContentList { - groupSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name) + volumeSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name) if err != nil { return nil, fmt.Errorf("failed to get group snapshot content %s from group snapshot content store: %v", contentRef.VolumeSnapshotContentRef.Name, err) } pvcVolumeSnapshotRefList = append(pvcVolumeSnapshotRefList, crdv1alpha1.PVCVolumeSnapshotPair{ VolumeSnapshotRef: v1.LocalObjectReference{ - Name: groupSnapshotContent.Spec.VolumeSnapshotRef.Name, + Name: volumeSnapshotContent.Spec.VolumeSnapshotRef.Name, + }, + PersistentVolumeClaimRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromPVName(contentRef.PersistentVolumeRef.Name).PVCName, }, }) } @@ -757,6 +780,7 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho return nil, err } var volumeHandles []string + var snapshotInfos utils.SnapshotInfoList for _, pv := range volumes { if pv.Spec.CSI == nil { err := fmt.Errorf( @@ -771,7 +795,19 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho ) return nil, err } - volumeHandles = append(volumeHandles, pv.Spec.CSI.VolumeHandle) + + pvcName := "" + if pv.Spec.ClaimRef != nil { + pvcName = pv.Spec.ClaimRef.Name + } + + volumeHandle := pv.Spec.CSI.VolumeHandle + volumeHandles = append(volumeHandles, volumeHandle) + snapshotInfos = append(snapshotInfos, utils.SnapshotInfo{ + VolumeHandle: volumeHandle, + PVName: pv.Name, + PVCName: pvcName, + }) } groupSnapshotContent := &crdv1alpha1.VolumeGroupSnapshotContent{ @@ -789,6 +825,18 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho }, } + /* Add snapshot information as an annotation */ + jsonInfo, err := snapshotInfos.ToJSON() + if err != nil { + strerr := fmt.Errorf("Error while setting PV names annotation %s: %v", utils.GroupSnapshotKey(groupSnapshot), err) + return nil, newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), strerr.Error()) + } + + klog.V(5).Infof( + "createGroupSnapshotContent: set annotation [%s] on volume group snapshot content [%s].", + utils.AnnSnapshotInfo, utils.GroupSnapshotKey(groupSnapshot)) + metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo, jsonInfo) + /* Add secret reference details */ diff --git a/pkg/sidecar-controller/groupsnapshot_helper.go b/pkg/sidecar-controller/groupsnapshot_helper.go index c50bb13e6..a927d69b7 100644 --- a/pkg/sidecar-controller/groupsnapshot_helper.go +++ b/pkg/sidecar-controller/groupsnapshot_helper.go @@ -35,6 +35,13 @@ import ( "github.com/kubernetes-csi/external-snapshotter/v7/pkg/utils" ) +// snapshotInfo contains the basic information +// about a snapshotted volume +type snapshotInfo struct { + snapshotContentName string + volumeHandle string +} + func (ctrl *csiSnapshotSideCarController) storeGroupSnapshotContentUpdate(groupSnapshotContent interface{}) (bool, error) { return utils.StoreObjectUpdate(ctrl.groupSnapshotContentStore, groupSnapshotContent, "groupsnapshotcontent") } @@ -430,7 +437,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh return groupSnapshotContent, fmt.Errorf("failed to get secret reference for group snapshot content %s: %v", groupSnapshotContent.Name, err) } // Create individual snapshots and snapshot contents - var snapshotContentNames []string + var snapshotsInfo []snapshotInfo for _, snapshot := range snapshots { volumeSnapshotContentName := GetSnapshotContentNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId) volumeSnapshotName := GetSnapshotNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId) @@ -484,7 +491,10 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh if err != nil { return groupSnapshotContent, err } - snapshotContentNames = append(snapshotContentNames, vsc.Name) + snapshotsInfo = append(snapshotsInfo, snapshotInfo{ + snapshotContentName: vsc.Name, + volumeHandle: snapshot.SourceVolumeId, + }) _, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).Create(context.TODO(), volumeSnapshot, metav1.CreateOptions{}) if err != nil { @@ -497,7 +507,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh } } - newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentNames) + newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotsInfo) if err != nil { klog.Errorf("error updating status for volume group snapshot content %s: %v.", groupSnapshotContent.Name, err) return groupSnapshotContent, fmt.Errorf("error updating status for volume group snapshot content %s: %v", groupSnapshotContent.Name, err) @@ -633,7 +643,8 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( groupSnapshotHandle string, readyToUse bool, createdAt int64, - snapshotContentNames []string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { + snapshots []snapshotInfo, +) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { klog.V(5).Infof("updateGroupSnapshotContentStatus: updating VolumeGroupSnapshotContent [%s], groupSnapshotHandle %s, readyToUse %v, createdAt %v", groupSnapshotContent.Name, groupSnapshotHandle, readyToUse, createdAt) groupSnapshotContentObj, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Get(context.TODO(), groupSnapshotContent.Name, metav1.GetOptions{}) @@ -641,6 +652,25 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( return nil, fmt.Errorf("error get group snapshot content %s from api server: %v", groupSnapshotContent.Name, err) } + var snapshotInfoList utils.SnapshotInfoList + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) { + snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo]) + if err != nil { + klog.V(1).Infof( + "updateGroupSnapshotContentStatus[%s]: the content of the [%s] annotation is not valid: %s", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + err.Error(), + ) + } + } else { + klog.V(2).Infof( + "updateGroupSnapshotContentStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + ) + } + var newStatus *crdv1alpha1.VolumeGroupSnapshotContentStatus updated := false if groupSnapshotContentObj.Status == nil { @@ -649,10 +679,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( ReadyToUse: &readyToUse, CreationTime: &createdAt, } - for _, name := range snapshotContentNames { + for _, snapshot := range snapshots { newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{ VolumeSnapshotContentRef: v1.LocalObjectReference{ - Name: name, + Name: snapshot.snapshotContentName, + }, + PersistentVolumeRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromVolumeHandle(snapshot.volumeHandle).PVName, }, }) } @@ -675,10 +708,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( updated = true } if len(newStatus.PVVolumeSnapshotContentList) == 0 { - for _, name := range snapshotContentNames { + for _, snapshot := range snapshots { newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{ VolumeSnapshotContentRef: v1.LocalObjectReference{ - Name: name, + Name: snapshot.snapshotContentName, + }, + PersistentVolumeRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromVolumeHandle(snapshot.volumeHandle).PVName, }, }) } @@ -842,7 +878,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateGroupSnapshotContentStat } // TODO: Get a reference to snapshot contents for this volume group snapshot - updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []string{}) + updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []snapshotInfo{}) if err != nil { return groupSnapshotContent, err } diff --git a/pkg/utils/snapshot_info.go b/pkg/utils/snapshot_info.go new file mode 100644 index 000000000..754375449 --- /dev/null +++ b/pkg/utils/snapshot_info.go @@ -0,0 +1,61 @@ +package utils + +import ( + "encoding/json" + "fmt" +) + +// SnapshotInfo contains basic information about a volume being snapshotted +type SnapshotInfo struct { + VolumeHandle string `json:"volumeHandle"` + PVName string `json:"pvName"` + PVCName string `json:"pvcName"` +} + +// SnapshotInfoList contains basic information about a set of volumes being snapshotted +type SnapshotInfoList []SnapshotInfo + +// ToJSON serizalizes to JSON a set of SnapshotInfo +func (data SnapshotInfoList) ToJSON() (string, error) { + result, err := json.Marshal(data) + if err != nil { + err = fmt.Errorf("while serializing SnapshotInfoList: %w", err) + } + return string(result), err +} + +// SnapshotInfoFromJSON deserializes from JSON a set of snapshot info +func SnapshotInfoFromJSON(content string) (SnapshotInfoList, error) { + var result SnapshotInfoList + + err := json.Unmarshal([]byte(content), &result) + if err != nil { + err = fmt.Errorf("while de-serializing SnapshotInfoList: %w", err) + } + + return result, err +} + +// GetFromVolumeHandle gets the entry from the list corresponding to a certain +// volume handle. Returns an empty SnapshotInfo if there is no such entry +func (data SnapshotInfoList) GetFromVolumeHandle(volumeHandle string) SnapshotInfo { + for i := range data { + if data[i].VolumeHandle == volumeHandle { + return data[i] + } + } + + return SnapshotInfo{} +} + +// GetFromPVName gets the entry from the list corresponding to a certain +// PV name. Returns an empty SnapshotInfo if there is no such entry +func (data SnapshotInfoList) GetFromPVName(pvName string) SnapshotInfo { + for i := range data { + if data[i].PVName == pvName { + return data[i] + } + } + + return SnapshotInfo{} +} diff --git a/pkg/utils/snapshot_info_test.go b/pkg/utils/snapshot_info_test.go new file mode 100644 index 000000000..d4a29535f --- /dev/null +++ b/pkg/utils/snapshot_info_test.go @@ -0,0 +1,129 @@ +package utils + +import ( + "reflect" + "testing" +) + +var ( + firstInfoListEntry = SnapshotInfo{ + VolumeHandle: "b98b9100-fbe2-11ee-b405-b2139ff66f78", + PVName: "pvc-f1718d88-e548-480b-bee8-cbfc47faaf59", + PVCName: "cluster-example-1", + } + + secondInfoListEntry = SnapshotInfo{ + VolumeHandle: "b98c3e51-fbe2-11ee-b405-b2139ff66f788", + PVName: "pvc-c59c9c0f-159a-43d6-9c60-61ccaf03158c", + PVCName: "cluster-example-1-wal", + } + + infoList = SnapshotInfoList{firstInfoListEntry, secondInfoListEntry} +) + +func TestMarshalUnmarshal(t *testing.T) { + jsonContent, err := infoList.ToJSON() + if err != nil { + t.Fatalf("JSON serialization failed: %v", err) + } + + unmarshalledInfoList, err := SnapshotInfoFromJSON(jsonContent) + if err != nil { + t.Fatalf("JSON deserialization failed: %v", err) + } + + if !reflect.DeepEqual(infoList, unmarshalledInfoList) { + t.Fatalf("unexpected info loss in serialization/deserialization: %v %v", infoList, unmarshalledInfoList) + } +} + +func TestEmptyMarshalUnmarshal(t *testing.T) { + var emptyInfoList SnapshotInfoList = nil + jsonContent, err := emptyInfoList.ToJSON() + if err != nil { + t.Fatalf("JSON serialization failed: %v", err) + } + + unmarshalledInfoList, err := SnapshotInfoFromJSON(jsonContent) + if err != nil { + t.Fatalf("JSON deserialization failed: %v", err) + } + + if !reflect.DeepEqual(emptyInfoList, unmarshalledInfoList) { + t.Fatalf("unexpected info loss in serialization/deserialization: %v %v", infoList, unmarshalledInfoList) + } +} + +func TestGetFromVolumeHandle(t *testing.T) { + testcases := []struct { + volumeHandle string + result SnapshotInfo + infoList SnapshotInfoList + }{ + { + volumeHandle: firstInfoListEntry.VolumeHandle, + result: firstInfoListEntry, + infoList: infoList, + }, + { + volumeHandle: secondInfoListEntry.VolumeHandle, + result: secondInfoListEntry, + infoList: infoList, + }, + { + volumeHandle: "", + result: SnapshotInfo{}, + infoList: infoList, + }, + { + volumeHandle: "", + result: SnapshotInfo{}, + infoList: nil, + }, + } + + for _, tc := range testcases { + t.Logf("looking for %s:", tc.volumeHandle) + result := tc.infoList.GetFromVolumeHandle(tc.volumeHandle) + if !reflect.DeepEqual(result, tc.result) { + t.Fatalf("unexpected GetFromVolumeHandle result: %v %v", result, tc.result) + } + } +} + +func TestGetFromPVName(t *testing.T) { + testcases := []struct { + pvName string + result SnapshotInfo + infoList SnapshotInfoList + }{ + { + pvName: firstInfoListEntry.PVName, + result: firstInfoListEntry, + infoList: infoList, + }, + { + pvName: secondInfoListEntry.PVName, + result: secondInfoListEntry, + infoList: infoList, + }, + { + pvName: "", + result: SnapshotInfo{}, + infoList: infoList, + }, + { + pvName: "", + result: SnapshotInfo{}, + infoList: nil, + }, + } + + for _, tc := range testcases { + t.Logf("looking for %s:", tc.pvName) + result := tc.infoList.GetFromPVName(tc.pvName) + if !reflect.DeepEqual(result, tc.result) { + t.Fatalf("unexpected GetFromPVName result: %v %v", result, tc.result) + } + } +} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index d893ec983..b83a3f16d 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -143,6 +143,9 @@ const ( AnnDeletionGroupSecretRefName = "groupsnapshot.storage.kubernetes.io/deletion-secret-name" AnnDeletionGroupSecretRefNamespace = "groupsnapshot.storage.kubernetes.io/deletion-secret-namespace" + // Annotation to store the basic information of a group snapshot + AnnSnapshotInfo = "groupsnapshot.storage.kubernetes.io/info" + // VolumeSnapshotContentInvalidLabel is applied to invalid content as a label key. The value does not matter. // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/177-volume-snapshot/tighten-validation-webhook-crd.md#automatic-labelling-of-invalid-objects VolumeSnapshotContentInvalidLabel = "snapshot.storage.kubernetes.io/invalid-snapshot-content-resource"