diff --git a/go.mod b/go.mod index 35153f44f..39e0a21fc 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/container-storage-interface/spec v1.9.0 github.com/evanphx/json-patch v5.9.0+incompatible github.com/fsnotify/fsnotify v1.7.0 + github.com/go-openapi/jsonpointer v0.21.0 github.com/golang/mock v1.6.0 github.com/google/gofuzz v1.2.0 github.com/kubernetes-csi/csi-lib-utils v0.18.0 @@ -33,7 +34,6 @@ require ( github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/pkg/common-controller/groupsnapshot_controller_helper.go b/pkg/common-controller/groupsnapshot_controller_helper.go index c66ff4814..f26406266 100644 --- a/pkg/common-controller/groupsnapshot_controller_helper.go +++ b/pkg/common-controller/groupsnapshot_controller_helper.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/go-openapi/jsonpointer" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -564,16 +565,39 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotStatus(groupSnapshot volumeSnapshotErr = groupSnapshotContent.Status.Error.DeepCopy() } + var snapshotInfoList utils.SnapshotInfoList + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) { + var err error + snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo]) + if err != nil { + klog.V(1).Infof( + "updateGroupSnapshotStatus[%s]: the content of the [%s] annotation is not valid: %s", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + err.Error(), + ) + } + } else { + klog.V(2).Infof( + "updateGroupSnapshotStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + ) + } + var pvcVolumeSnapshotRefList []crdv1alpha1.PVCVolumeSnapshotPair if groupSnapshotContent.Status != nil && len(groupSnapshotContent.Status.PVVolumeSnapshotContentList) != 0 { for _, contentRef := range groupSnapshotContent.Status.PVVolumeSnapshotContentList { - groupSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name) + volumeSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name) if err != nil { return nil, fmt.Errorf("failed to get group snapshot content %s from group snapshot content store: %v", contentRef.VolumeSnapshotContentRef.Name, err) } pvcVolumeSnapshotRefList = append(pvcVolumeSnapshotRefList, crdv1alpha1.PVCVolumeSnapshotPair{ VolumeSnapshotRef: v1.LocalObjectReference{ - Name: groupSnapshotContent.Spec.VolumeSnapshotRef.Name, + Name: volumeSnapshotContent.Spec.VolumeSnapshotRef.Name, + }, + PersistentVolumeClaimRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromPVName(contentRef.PersistentVolumeRef.Name).PVCName, }, }) } @@ -649,6 +673,29 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotStatus(groupSnapshot if !utils.IsGroupSnapshotReady(groupSnapshotObj) && utils.IsGroupSnapshotReady(groupSnapshotClone) { msg := fmt.Sprintf("GroupSnapshot %s is ready to use.", utils.GroupSnapshotKey(groupSnapshot)) ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeNormal, "GroupSnapshotReady", msg) + + // The VolumeGroupSnapshot resource is now ready to use. There's no need for the + // groupsnapshot.storage.kubernetes.io/info annotation anymore so we can remove it. + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) { + klog.V(5).Infof("updateGroupSnapshotStatus[%s]: removing info annotation on VolumeGroupSnapshotContent[%s]", + groupSnapshot.Name, + groupSnapshotContent.Name) + patches := []utils.PatchOp{ + { + Op: "remove", + Path: fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(utils.AnnSnapshotInfo)), + }, + } + + _, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset) + if err != nil { + klog.V(5).Infof("updateGroupSnapshotStatus[%s]: error while removing info annotation on VolumeGroupSnapshotContent[%s]: %s", + groupSnapshot.Name, + groupSnapshotContent.Name, + err.Error()) + return nil, newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error()) + } + } } newGroupSnapshotObj, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).UpdateStatus(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{}) @@ -757,6 +804,7 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho return nil, err } var volumeHandles []string + var snapshotInfos utils.SnapshotInfoList for _, pv := range volumes { if pv.Spec.CSI == nil { err := fmt.Errorf( @@ -771,7 +819,19 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho ) return nil, err } - volumeHandles = append(volumeHandles, pv.Spec.CSI.VolumeHandle) + + pvcName := "" + if pv.Spec.ClaimRef != nil { + pvcName = pv.Spec.ClaimRef.Name + } + + volumeHandle := pv.Spec.CSI.VolumeHandle + volumeHandles = append(volumeHandles, volumeHandle) + snapshotInfos = append(snapshotInfos, utils.SnapshotInfo{ + VolumeHandle: volumeHandle, + PVName: pv.Name, + PVCName: pvcName, + }) } groupSnapshotContent := &crdv1alpha1.VolumeGroupSnapshotContent{ @@ -789,6 +849,18 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho }, } + /* Add snapshot information as an annotation */ + jsonInfo, err := snapshotInfos.ToJSON() + if err != nil { + strerr := fmt.Errorf("Error while setting PV names annotation %s: %v", utils.GroupSnapshotKey(groupSnapshot), err) + return nil, newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), strerr.Error()) + } + + klog.V(5).Infof( + "createGroupSnapshotContent: set annotation [%s] on volume group snapshot content [%s].", + utils.AnnSnapshotInfo, utils.GroupSnapshotKey(groupSnapshot)) + metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo, jsonInfo) + /* Add secret reference details */ diff --git a/pkg/sidecar-controller/groupsnapshot_helper.go b/pkg/sidecar-controller/groupsnapshot_helper.go index c50bb13e6..66d7983d8 100644 --- a/pkg/sidecar-controller/groupsnapshot_helper.go +++ b/pkg/sidecar-controller/groupsnapshot_helper.go @@ -35,6 +35,13 @@ import ( "github.com/kubernetes-csi/external-snapshotter/v7/pkg/utils" ) +// snapshotContentNameVolumeHandlePair represent the link between a VolumeSnapshotContent and +// the handle of the volume that was snapshotted +type snapshotContentNameVolumeHandlePair struct { + snapshotContentName string + volumeHandle string +} + func (ctrl *csiSnapshotSideCarController) storeGroupSnapshotContentUpdate(groupSnapshotContent interface{}) (bool, error) { return utils.StoreObjectUpdate(ctrl.groupSnapshotContentStore, groupSnapshotContent, "groupsnapshotcontent") } @@ -430,7 +437,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh return groupSnapshotContent, fmt.Errorf("failed to get secret reference for group snapshot content %s: %v", groupSnapshotContent.Name, err) } // Create individual snapshots and snapshot contents - var snapshotContentNames []string + var snapshotContentLinks []snapshotContentNameVolumeHandlePair for _, snapshot := range snapshots { volumeSnapshotContentName := GetSnapshotContentNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId) volumeSnapshotName := GetSnapshotNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId) @@ -484,7 +491,10 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh if err != nil { return groupSnapshotContent, err } - snapshotContentNames = append(snapshotContentNames, vsc.Name) + snapshotContentLinks = append(snapshotContentLinks, snapshotContentNameVolumeHandlePair{ + snapshotContentName: vsc.Name, + volumeHandle: snapshot.SourceVolumeId, + }) _, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).Create(context.TODO(), volumeSnapshot, metav1.CreateOptions{}) if err != nil { @@ -497,7 +507,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh } } - newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentNames) + newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentLinks) if err != nil { klog.Errorf("error updating status for volume group snapshot content %s: %v.", groupSnapshotContent.Name, err) return groupSnapshotContent, fmt.Errorf("error updating status for volume group snapshot content %s: %v", groupSnapshotContent.Name, err) @@ -633,7 +643,8 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( groupSnapshotHandle string, readyToUse bool, createdAt int64, - snapshotContentNames []string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { + snapshotContentLinks []snapshotContentNameVolumeHandlePair, +) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { klog.V(5).Infof("updateGroupSnapshotContentStatus: updating VolumeGroupSnapshotContent [%s], groupSnapshotHandle %s, readyToUse %v, createdAt %v", groupSnapshotContent.Name, groupSnapshotHandle, readyToUse, createdAt) groupSnapshotContentObj, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Get(context.TODO(), groupSnapshotContent.Name, metav1.GetOptions{}) @@ -641,6 +652,25 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( return nil, fmt.Errorf("error get group snapshot content %s from api server: %v", groupSnapshotContent.Name, err) } + var snapshotInfoList utils.SnapshotInfoList + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) { + snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo]) + if err != nil { + klog.V(1).Infof( + "updateGroupSnapshotContentStatus[%s]: the content of the [%s] annotation is not valid: %s", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + err.Error(), + ) + } + } else { + klog.V(2).Infof( + "updateGroupSnapshotContentStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs", + groupSnapshotContent.Name, + utils.AnnSnapshotInfo, + ) + } + var newStatus *crdv1alpha1.VolumeGroupSnapshotContentStatus updated := false if groupSnapshotContentObj.Status == nil { @@ -649,10 +679,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( ReadyToUse: &readyToUse, CreationTime: &createdAt, } - for _, name := range snapshotContentNames { + for _, snapshotContentLink := range snapshotContentLinks { newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{ VolumeSnapshotContentRef: v1.LocalObjectReference{ - Name: name, + Name: snapshotContentLink.snapshotContentName, + }, + PersistentVolumeRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromVolumeHandle(snapshotContentLink.volumeHandle).PVName, }, }) } @@ -675,10 +708,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( updated = true } if len(newStatus.PVVolumeSnapshotContentList) == 0 { - for _, name := range snapshotContentNames { + for _, snapshotContentLink := range snapshotContentLinks { newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{ VolumeSnapshotContentRef: v1.LocalObjectReference{ - Name: name, + Name: snapshotContentLink.snapshotContentName, + }, + PersistentVolumeRef: v1.LocalObjectReference{ + Name: snapshotInfoList.GetFromVolumeHandle(snapshotContentLink.volumeHandle).PVName, }, }) } @@ -842,7 +878,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateGroupSnapshotContentStat } // TODO: Get a reference to snapshot contents for this volume group snapshot - updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []string{}) + updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []snapshotContentNameVolumeHandlePair{}) if err != nil { return groupSnapshotContent, err } diff --git a/pkg/utils/snapshot_info.go b/pkg/utils/snapshot_info.go new file mode 100644 index 000000000..754375449 --- /dev/null +++ b/pkg/utils/snapshot_info.go @@ -0,0 +1,61 @@ +package utils + +import ( + "encoding/json" + "fmt" +) + +// SnapshotInfo contains basic information about a volume being snapshotted +type SnapshotInfo struct { + VolumeHandle string `json:"volumeHandle"` + PVName string `json:"pvName"` + PVCName string `json:"pvcName"` +} + +// SnapshotInfoList contains basic information about a set of volumes being snapshotted +type SnapshotInfoList []SnapshotInfo + +// ToJSON serizalizes to JSON a set of SnapshotInfo +func (data SnapshotInfoList) ToJSON() (string, error) { + result, err := json.Marshal(data) + if err != nil { + err = fmt.Errorf("while serializing SnapshotInfoList: %w", err) + } + return string(result), err +} + +// SnapshotInfoFromJSON deserializes from JSON a set of snapshot info +func SnapshotInfoFromJSON(content string) (SnapshotInfoList, error) { + var result SnapshotInfoList + + err := json.Unmarshal([]byte(content), &result) + if err != nil { + err = fmt.Errorf("while de-serializing SnapshotInfoList: %w", err) + } + + return result, err +} + +// GetFromVolumeHandle gets the entry from the list corresponding to a certain +// volume handle. Returns an empty SnapshotInfo if there is no such entry +func (data SnapshotInfoList) GetFromVolumeHandle(volumeHandle string) SnapshotInfo { + for i := range data { + if data[i].VolumeHandle == volumeHandle { + return data[i] + } + } + + return SnapshotInfo{} +} + +// GetFromPVName gets the entry from the list corresponding to a certain +// PV name. Returns an empty SnapshotInfo if there is no such entry +func (data SnapshotInfoList) GetFromPVName(pvName string) SnapshotInfo { + for i := range data { + if data[i].PVName == pvName { + return data[i] + } + } + + return SnapshotInfo{} +} diff --git a/pkg/utils/snapshot_info_test.go b/pkg/utils/snapshot_info_test.go new file mode 100644 index 000000000..d4a29535f --- /dev/null +++ b/pkg/utils/snapshot_info_test.go @@ -0,0 +1,129 @@ +package utils + +import ( + "reflect" + "testing" +) + +var ( + firstInfoListEntry = SnapshotInfo{ + VolumeHandle: "b98b9100-fbe2-11ee-b405-b2139ff66f78", + PVName: "pvc-f1718d88-e548-480b-bee8-cbfc47faaf59", + PVCName: "cluster-example-1", + } + + secondInfoListEntry = SnapshotInfo{ + VolumeHandle: "b98c3e51-fbe2-11ee-b405-b2139ff66f788", + PVName: "pvc-c59c9c0f-159a-43d6-9c60-61ccaf03158c", + PVCName: "cluster-example-1-wal", + } + + infoList = SnapshotInfoList{firstInfoListEntry, secondInfoListEntry} +) + +func TestMarshalUnmarshal(t *testing.T) { + jsonContent, err := infoList.ToJSON() + if err != nil { + t.Fatalf("JSON serialization failed: %v", err) + } + + unmarshalledInfoList, err := SnapshotInfoFromJSON(jsonContent) + if err != nil { + t.Fatalf("JSON deserialization failed: %v", err) + } + + if !reflect.DeepEqual(infoList, unmarshalledInfoList) { + t.Fatalf("unexpected info loss in serialization/deserialization: %v %v", infoList, unmarshalledInfoList) + } +} + +func TestEmptyMarshalUnmarshal(t *testing.T) { + var emptyInfoList SnapshotInfoList = nil + jsonContent, err := emptyInfoList.ToJSON() + if err != nil { + t.Fatalf("JSON serialization failed: %v", err) + } + + unmarshalledInfoList, err := SnapshotInfoFromJSON(jsonContent) + if err != nil { + t.Fatalf("JSON deserialization failed: %v", err) + } + + if !reflect.DeepEqual(emptyInfoList, unmarshalledInfoList) { + t.Fatalf("unexpected info loss in serialization/deserialization: %v %v", infoList, unmarshalledInfoList) + } +} + +func TestGetFromVolumeHandle(t *testing.T) { + testcases := []struct { + volumeHandle string + result SnapshotInfo + infoList SnapshotInfoList + }{ + { + volumeHandle: firstInfoListEntry.VolumeHandle, + result: firstInfoListEntry, + infoList: infoList, + }, + { + volumeHandle: secondInfoListEntry.VolumeHandle, + result: secondInfoListEntry, + infoList: infoList, + }, + { + volumeHandle: "", + result: SnapshotInfo{}, + infoList: infoList, + }, + { + volumeHandle: "", + result: SnapshotInfo{}, + infoList: nil, + }, + } + + for _, tc := range testcases { + t.Logf("looking for %s:", tc.volumeHandle) + result := tc.infoList.GetFromVolumeHandle(tc.volumeHandle) + if !reflect.DeepEqual(result, tc.result) { + t.Fatalf("unexpected GetFromVolumeHandle result: %v %v", result, tc.result) + } + } +} + +func TestGetFromPVName(t *testing.T) { + testcases := []struct { + pvName string + result SnapshotInfo + infoList SnapshotInfoList + }{ + { + pvName: firstInfoListEntry.PVName, + result: firstInfoListEntry, + infoList: infoList, + }, + { + pvName: secondInfoListEntry.PVName, + result: secondInfoListEntry, + infoList: infoList, + }, + { + pvName: "", + result: SnapshotInfo{}, + infoList: infoList, + }, + { + pvName: "", + result: SnapshotInfo{}, + infoList: nil, + }, + } + + for _, tc := range testcases { + t.Logf("looking for %s:", tc.pvName) + result := tc.infoList.GetFromPVName(tc.pvName) + if !reflect.DeepEqual(result, tc.result) { + t.Fatalf("unexpected GetFromPVName result: %v %v", result, tc.result) + } + } +} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index d893ec983..b83a3f16d 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -143,6 +143,9 @@ const ( AnnDeletionGroupSecretRefName = "groupsnapshot.storage.kubernetes.io/deletion-secret-name" AnnDeletionGroupSecretRefNamespace = "groupsnapshot.storage.kubernetes.io/deletion-secret-namespace" + // Annotation to store the basic information of a group snapshot + AnnSnapshotInfo = "groupsnapshot.storage.kubernetes.io/info" + // VolumeSnapshotContentInvalidLabel is applied to invalid content as a label key. The value does not matter. // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/177-volume-snapshot/tighten-validation-webhook-crd.md#automatic-labelling-of-invalid-objects VolumeSnapshotContentInvalidLabel = "snapshot.storage.kubernetes.io/invalid-snapshot-content-resource"