Skip to content

Commit

Permalink
Link PVCs and PVs in VolumeGroupSnapshot and VolumeGroupSnapshotContent
Browse files Browse the repository at this point in the history
This use the update API to set `persistentVolumeClaimRef` in
`VolumeGroupSnapshot` and `persistentVolumeName` in
`VolumeGroupSnapshotContent` to the corresponding objects.

This makes restoring volumes from a VolumeGroupSnapshot easier.

Related: #969
  • Loading branch information
leonardoce committed May 15, 2024
1 parent 22117ab commit bb3b87a
Show file tree
Hide file tree
Showing 6 changed files with 314 additions and 13 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/container-storage-interface/spec v1.9.0
github.com/evanphx/json-patch v5.9.0+incompatible
github.com/fsnotify/fsnotify v1.7.0
github.com/go-openapi/jsonpointer v0.21.0
github.com/golang/mock v1.6.0
github.com/google/gofuzz v1.2.0
github.com/kubernetes-csi/csi-lib-utils v0.18.0
Expand Down Expand Up @@ -33,7 +34,6 @@ require (
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
78 changes: 75 additions & 3 deletions pkg/common-controller/groupsnapshot_controller_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"time"

"github.com/go-openapi/jsonpointer"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrs "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -564,16 +565,39 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotStatus(groupSnapshot
volumeSnapshotErr = groupSnapshotContent.Status.Error.DeepCopy()
}

var snapshotInfoList utils.SnapshotInfoList
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) {
var err error
snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo])
if err != nil {
klog.V(1).Infof(
"updateGroupSnapshotStatus[%s]: the content of the [%s] annotation is not valid: %s",
groupSnapshotContent.Name,
utils.AnnSnapshotInfo,
err.Error(),
)
}
} else {
klog.V(2).Infof(
"updateGroupSnapshotStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs",
groupSnapshotContent.Name,
utils.AnnSnapshotInfo,
)
}

var pvcVolumeSnapshotRefList []crdv1alpha1.PVCVolumeSnapshotPair
if groupSnapshotContent.Status != nil && len(groupSnapshotContent.Status.PVVolumeSnapshotContentList) != 0 {
for _, contentRef := range groupSnapshotContent.Status.PVVolumeSnapshotContentList {
groupSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name)
volumeSnapshotContent, err := ctrl.contentLister.Get(contentRef.VolumeSnapshotContentRef.Name)
if err != nil {
return nil, fmt.Errorf("failed to get group snapshot content %s from group snapshot content store: %v", contentRef.VolumeSnapshotContentRef.Name, err)
}
pvcVolumeSnapshotRefList = append(pvcVolumeSnapshotRefList, crdv1alpha1.PVCVolumeSnapshotPair{
VolumeSnapshotRef: v1.LocalObjectReference{
Name: groupSnapshotContent.Spec.VolumeSnapshotRef.Name,
Name: volumeSnapshotContent.Spec.VolumeSnapshotRef.Name,
},
PersistentVolumeClaimRef: v1.LocalObjectReference{
Name: snapshotInfoList.GetFromPVName(contentRef.PersistentVolumeRef.Name).PVCName,
},
})
}
Expand Down Expand Up @@ -649,6 +673,29 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotStatus(groupSnapshot
if !utils.IsGroupSnapshotReady(groupSnapshotObj) && utils.IsGroupSnapshotReady(groupSnapshotClone) {
msg := fmt.Sprintf("GroupSnapshot %s is ready to use.", utils.GroupSnapshotKey(groupSnapshot))
ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeNormal, "GroupSnapshotReady", msg)

// The VolumeGroupSnapshot resource is now ready to use. There's no need for the
// groupsnapshot.storage.kubernetes.io/info annotation anymore so we can remove it.
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) {
klog.V(5).Infof("updateGroupSnapshotStatus[%s]: removing info annotation on VolumeGroupSnapshotContent[%s]",
groupSnapshot.Name,
groupSnapshotContent.Name)
patches := []utils.PatchOp{
{
Op: "remove",
Path: fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(utils.AnnSnapshotInfo)),
},
}

_, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset)
if err != nil {
klog.V(5).Infof("updateGroupSnapshotStatus[%s]: error while removing info annotation on VolumeGroupSnapshotContent[%s]: %s",
groupSnapshot.Name,
groupSnapshotContent.Name,
err.Error())
return nil, newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error())
}
}
}

newGroupSnapshotObj, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).UpdateStatus(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{})
Expand Down Expand Up @@ -757,6 +804,7 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
return nil, err
}
var volumeHandles []string
var snapshotInfos utils.SnapshotInfoList
for _, pv := range volumes {
if pv.Spec.CSI == nil {
err := fmt.Errorf(
Expand All @@ -771,7 +819,19 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
)
return nil, err
}
volumeHandles = append(volumeHandles, pv.Spec.CSI.VolumeHandle)

pvcName := ""
if pv.Spec.ClaimRef != nil {
pvcName = pv.Spec.ClaimRef.Name
}

volumeHandle := pv.Spec.CSI.VolumeHandle
volumeHandles = append(volumeHandles, volumeHandle)
snapshotInfos = append(snapshotInfos, utils.SnapshotInfo{
VolumeHandle: volumeHandle,
PVName: pv.Name,
PVCName: pvcName,
})
}

groupSnapshotContent := &crdv1alpha1.VolumeGroupSnapshotContent{
Expand All @@ -789,6 +849,18 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
},
}

/* Add snapshot information as an annotation */
jsonInfo, err := snapshotInfos.ToJSON()
if err != nil {
strerr := fmt.Errorf("Error while setting PV names annotation %s: %v", utils.GroupSnapshotKey(groupSnapshot), err)
return nil, newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), strerr.Error())
}

klog.V(5).Infof(
"createGroupSnapshotContent: set annotation [%s] on volume group snapshot content [%s].",
utils.AnnSnapshotInfo, utils.GroupSnapshotKey(groupSnapshot))
metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo, jsonInfo)

/*
Add secret reference details
*/
Expand Down
54 changes: 45 additions & 9 deletions pkg/sidecar-controller/groupsnapshot_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ import (
"github.com/kubernetes-csi/external-snapshotter/v7/pkg/utils"
)

// snapshotContentNameVolumeHandlePair represent the link between a VolumeSnapshotContent and
// the handle of the volume that was snapshotted
type snapshotContentNameVolumeHandlePair struct {
snapshotContentName string
volumeHandle string
}

func (ctrl *csiSnapshotSideCarController) storeGroupSnapshotContentUpdate(groupSnapshotContent interface{}) (bool, error) {
return utils.StoreObjectUpdate(ctrl.groupSnapshotContentStore, groupSnapshotContent, "groupsnapshotcontent")
}
Expand Down Expand Up @@ -430,7 +437,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
return groupSnapshotContent, fmt.Errorf("failed to get secret reference for group snapshot content %s: %v", groupSnapshotContent.Name, err)
}
// Create individual snapshots and snapshot contents
var snapshotContentNames []string
var snapshotContentLinks []snapshotContentNameVolumeHandlePair
for _, snapshot := range snapshots {
volumeSnapshotContentName := GetSnapshotContentNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId)
volumeSnapshotName := GetSnapshotNameForVolumeGroupSnapshotContent(string(groupSnapshotContent.UID), snapshot.SourceVolumeId)
Expand Down Expand Up @@ -484,7 +491,10 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
if err != nil {
return groupSnapshotContent, err
}
snapshotContentNames = append(snapshotContentNames, vsc.Name)
snapshotContentLinks = append(snapshotContentLinks, snapshotContentNameVolumeHandlePair{
snapshotContentName: vsc.Name,
volumeHandle: snapshot.SourceVolumeId,
})

_, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).Create(context.TODO(), volumeSnapshot, metav1.CreateOptions{})
if err != nil {
Expand All @@ -497,7 +507,7 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
}
}

newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentNames)
newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentLinks)
if err != nil {
klog.Errorf("error updating status for volume group snapshot content %s: %v.", groupSnapshotContent.Name, err)
return groupSnapshotContent, fmt.Errorf("error updating status for volume group snapshot content %s: %v", groupSnapshotContent.Name, err)
Expand Down Expand Up @@ -633,14 +643,34 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus(
groupSnapshotHandle string,
readyToUse bool,
createdAt int64,
snapshotContentNames []string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
snapshotContentLinks []snapshotContentNameVolumeHandlePair,
) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
klog.V(5).Infof("updateGroupSnapshotContentStatus: updating VolumeGroupSnapshotContent [%s], groupSnapshotHandle %s, readyToUse %v, createdAt %v", groupSnapshotContent.Name, groupSnapshotHandle, readyToUse, createdAt)

groupSnapshotContentObj, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Get(context.TODO(), groupSnapshotContent.Name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("error get group snapshot content %s from api server: %v", groupSnapshotContent.Name, err)
}

var snapshotInfoList utils.SnapshotInfoList
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnSnapshotInfo) {
snapshotInfoList, err = utils.SnapshotInfoFromJSON(groupSnapshotContent.Annotations[utils.AnnSnapshotInfo])
if err != nil {
klog.V(1).Infof(
"updateGroupSnapshotContentStatus[%s]: the content of the [%s] annotation is not valid: %s",
groupSnapshotContent.Name,
utils.AnnSnapshotInfo,
err.Error(),
)
}
} else {
klog.V(2).Infof(
"updateGroupSnapshotContentStatus[%s]: the [%s] annotation is empty, we won't be able to associate PVs",
groupSnapshotContent.Name,
utils.AnnSnapshotInfo,
)
}

var newStatus *crdv1alpha1.VolumeGroupSnapshotContentStatus
updated := false
if groupSnapshotContentObj.Status == nil {
Expand All @@ -649,10 +679,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus(
ReadyToUse: &readyToUse,
CreationTime: &createdAt,
}
for _, name := range snapshotContentNames {
for _, snapshotContentLink := range snapshotContentLinks {
newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{
VolumeSnapshotContentRef: v1.LocalObjectReference{
Name: name,
Name: snapshotContentLink.snapshotContentName,
},
PersistentVolumeRef: v1.LocalObjectReference{
Name: snapshotInfoList.GetFromVolumeHandle(snapshotContentLink.volumeHandle).PVName,
},
})
}
Expand All @@ -675,10 +708,13 @@ func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus(
updated = true
}
if len(newStatus.PVVolumeSnapshotContentList) == 0 {
for _, name := range snapshotContentNames {
for _, snapshotContentLink := range snapshotContentLinks {
newStatus.PVVolumeSnapshotContentList = append(newStatus.PVVolumeSnapshotContentList, crdv1alpha1.PVVolumeSnapshotContentPair{
VolumeSnapshotContentRef: v1.LocalObjectReference{
Name: name,
Name: snapshotContentLink.snapshotContentName,
},
PersistentVolumeRef: v1.LocalObjectReference{
Name: snapshotInfoList.GetFromVolumeHandle(snapshotContentLink.volumeHandle).PVName,
},
})
}
Expand Down Expand Up @@ -842,7 +878,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateGroupSnapshotContentStat
}

// TODO: Get a reference to snapshot contents for this volume group snapshot
updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []string{})
updatedContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), []snapshotContentNameVolumeHandlePair{})
if err != nil {
return groupSnapshotContent, err
}
Expand Down
61 changes: 61 additions & 0 deletions pkg/utils/snapshot_info.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package utils

import (
"encoding/json"
"fmt"
)

// SnapshotInfo contains basic information about a volume being snapshotted
type SnapshotInfo struct {
VolumeHandle string `json:"volumeHandle"`
PVName string `json:"pvName"`
PVCName string `json:"pvcName"`
}

// SnapshotInfoList contains basic information about a set of volumes being snapshotted
type SnapshotInfoList []SnapshotInfo

// ToJSON serizalizes to JSON a set of SnapshotInfo
func (data SnapshotInfoList) ToJSON() (string, error) {
result, err := json.Marshal(data)
if err != nil {
err = fmt.Errorf("while serializing SnapshotInfoList: %w", err)
}
return string(result), err
}

// SnapshotInfoFromJSON deserializes from JSON a set of snapshot info
func SnapshotInfoFromJSON(content string) (SnapshotInfoList, error) {
var result SnapshotInfoList

err := json.Unmarshal([]byte(content), &result)
if err != nil {
err = fmt.Errorf("while de-serializing SnapshotInfoList: %w", err)
}

return result, err
}

// GetFromVolumeHandle gets the entry from the list corresponding to a certain
// volume handle. Returns an empty SnapshotInfo if there is no such entry
func (data SnapshotInfoList) GetFromVolumeHandle(volumeHandle string) SnapshotInfo {
for i := range data {
if data[i].VolumeHandle == volumeHandle {
return data[i]
}
}

return SnapshotInfo{}
}

// GetFromPVName gets the entry from the list corresponding to a certain
// PV name. Returns an empty SnapshotInfo if there is no such entry
func (data SnapshotInfoList) GetFromPVName(pvName string) SnapshotInfo {
for i := range data {
if data[i].PVName == pvName {
return data[i]
}
}

return SnapshotInfo{}
}
Loading

0 comments on commit bb3b87a

Please sign in to comment.