[WIP] SnapshotGroup v1, with support for 1.25 (#150)

* rename v1beta1 to v1

* add v1 CRD

* update examples

* add support for beta1 CRD

* add import

* add debug info

* try update snapshotgroup

* update with json schema version

* more debug

* no create CRD by default

* add upgrade instructions

* add skip-crds note

.tool-versions

@@ -0,0 +1 @@
+golang 1.20

README.md

@@ -21,8 +21,6 @@ Gemini is a Kubernetes CRD and operator for managing `VolumeSnapshots`. This all
 to create a snapshot of the data on your `PersistentVolumes` on a regular schedule,
 retire old snapshots, and restore snapshots with minimal downtime.
 
-> Note: Like the VolumeSnapshot API it builds on, Gemini is **currently in beta**.
-
 ## Installation
 The Gemini Helm chart will install both the CRD and the operator into your cluster
 
@@ -49,6 +47,15 @@ kubectl api-resources | grep volumesnapshots
 Before getting started with Gemini, it's a good idea to make sure you're able to
 [create a VolumeSnapshot manually](https://kubernetes.io/docs/concepts/storage/volume-snapshots/#volumesnapshots).
 
+### Upgrading to V2
+Version 2.0 of Gemini updates the CRD from `v1beta1` to `v1`. There are no substantial
+changes, but `v1` adds better support for PersistentVolumeClaims on Kubernetes 1.25.
+
+If you want to keep the v1beta1 CRD available, you can run:
+```
+kubectl apply -f https://raw.githubusercontent.com/FairwindsOps/gemini/main/pkg/types/snapshotgroup/v1beta1/crd-with-beta1.yaml
+```
+before upgrading, and add `--skip-crds` when running `helm install`.
 
 ## Usage
 
@@ -112,7 +119,7 @@ E.g. right after a new snapshot is created, you'll see snapshots for
 
 
 #### Using an Existing PVC
-> See the [extended example](/examples/hackmd/README.md)
+> See the [extended example](/examples/codimd/README.md)
 The following example schedules snapshots every 10 minutes for a pre-existing PVC named `postgres`.
 
 ```yaml
@@ -200,7 +207,7 @@ $ kubectl scale all --all --replicas=1
 ```
 
 ## End-to-End Example
-To see gemini working end-to-end, check out [the HackMD example](examples/hackmd)
+To see gemini working end-to-end, check out [the CodiMD example](examples/codimd)
 
 ## Caveats
 * Like the VolumeSnapshot API it builds on, Gemini is **currently in beta**

examples/hackmd/README.md → examples/codimd/README.md

@@ -1,46 +1,31 @@
-# Gemini Example: HackMD
+# Gemini Example: CodiMD
 > Note: this will not work in a KIND cluster. It has been tested on DigitalOcean.
 
 ### Install the controller
 ```bash
-kubectl create ns gemini
 helm repo add fairwinds-stable https://charts.fairwinds.com/stable
-helm install gemini fairwinds-stable/gemini --namespace gemini
+helm install gemini fairwinds-stable/gemini --namespace gemini --create-namespace
 ```
 
-### Install HackMD
+### Install CodiMD
 ```bash
-kubectl create ns notepad
-helm install hackmd stable/hackmd -n notepad --version "2.0.*" --set postgresql.postgresqlPassword=thisisnotasecret
+helm repo add codimd https://helm.codimd.dev/
+helm upgrade --install codimd codimd/codimd -n codimd --create-namespace --set codimd.imageStorePersistentVolume.enabled=false
 ```
 
-This will create two PVCs, one for HackMD, and one for the Postgres instance that backs it.
+This will create a PVC for the Postgres instance that drives CodiMD.
 
 ### Set up the Backup Schedule
 ```bash
 cat <<EOF | kubectl apply -f -
 apiVersion: gemini.fairwinds.com/v1beta1
 kind: SnapshotGroup
 metadata:
-  name: hackmd
-  namespace: notepad
+  name: codimd-postgresql
+  namespace: codimd
 spec:
   persistentVolumeClaim:
-    claimName: hackmd
-  schedule:
-    - every: "10 minutes"
-      keep: 3
-    - every: hour
-      keep: 1
----
-apiVersion: gemini.fairwinds.com/v1beta1
-kind: SnapshotGroup
-metadata:
-  name: hackmd-postgresql
-  namespace: notepad
-spec:
-  persistentVolumeClaim:
-    claimName: data-hackmd-postgresql-0
+    claimName: data-codimd-postgresql-0
   schedule:
     - every: "10 minutes"
       keep: 3
@@ -49,69 +34,66 @@ spec:
 EOF
 ```
 
-within 30 seconds or so, you should see a couple `VolumeSnapshots`:
+within 30 seconds or so, you should see a `VolumeSnapshot`:
 ```bash
-$ kubectl get volumesnapshot -n notepad
+$ kubectl get volumesnapshot -n codimd
 NAME                           READYTOUSE   SOURCEPVC                  SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS      SNAPSHOTCONTENT                                    CREATIONTIME   AGE
-hackmd-1594929306              true         hackmd                                             2Gi           do-block-storage   snapcontent-2dac493e-116e-41b7-9ca2-cb797ac7c40b   17s            19s
-hackmd-postgresql-1594929307   true         data-hackmd-postgresql-0                           8Gi           do-block-storage   snapcontent-300e10a1-945a-483f-a461-9b073c853ddf   16s            18s
+codimd-postgresql-1677262979   true         data-codimd-postgresql-0                           8Gi           do-block-storage   snapcontent-3aff5b51-e0c1-4154-bd93-4b396bf0c16e   12m            12m
 ```
 
 ### Create a document
 ```bash
-kubectl port-forward svc/hackmd 3000:3000 -n notepad
+kubectl port-forward svc/codimd 3000:80 -n codimd
 ```
 
-Visit `localhost:3000` and create a new guest document. Enter some dummy text.
+Visit `localhost:3000` and sign up for an account. (You can use a dummy email. Make sure to click `Register` instead of hitting enter when first signing up.) Create a new note and enter some text.
 
 ### Trigger a backup
 Rather than waiting for Gemini to create the next backup, you can delete existing
 backups, and Gemini will create a new one.
 
 ```bash
-kubectl delete volumesnapshot --all -n notepad
+kubectl delete volumesnapshot --all -n codimd
 ```
 
 Within 30 seconds, you should see new snapshots appear. Make sure to wait until `READYTOUSE` is true
 ```bash
-$ kubectl get volumesnapshot -n notepad
+$ kubectl get volumesnapshot -n codimd
 NAME                           READYTOUSE   SOURCEPVC                  SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS      SNAPSHOTCONTENT                                    CREATIONTIME   AGE
-hackmd-1594929516              true         hackmd                                             2Gi           do-block-storage   snapcontent-e75421c6-c4ca-4bbf-81f4-a2fb0706b957   5s             7s
-hackmd-postgresql-1594929517   true         data-hackmd-postgresql-0                           8Gi           do-block-storage   snapcontent-ad71c1f8-af7b-4cdc-85ba-e512a77095a3   4s             6s
+codimd-1594929516              true         codimd                                             2Gi           do-block-storage   snapcontent-e75421c6-c4ca-4bbf-81f4-a2fb0706b957   5s             7s
+codimd-postgresql-1594929517   true         data-codimd-postgresql-0                           8Gi           do-block-storage   snapcontent-ad71c1f8-af7b-4cdc-85ba-e512a77095a3   4s             6s
 ```
 
 ### Edit your document again
 ```bash
-kubectl port-forward svc/hackmd 3000:3000 -n notepad
+kubectl port-forward svc/codimd 3000:80 -n codimd
 ```
 
 Go back to your document (if you lost it, you can find it again by going to
 `localhost:3000` and clicking `History`).
 
-Make some more dummy edits. These will get deleted when we restore.
+Delete what you wrote and replace it with something else. These changes will get reverted when we restore.
 
 ### Perform the restore
-> Note that we only need to restore PostgreSQL - we didn't change anything in the core app.
-
 First, we need to scale down our deployment. We can't swap out a PVC in-place,
 so you'll necessarily incur some downtime.
 
 ```bash
-kubectl scale all --all --replicas=0 -n notepad
+kubectl scale all --all --replicas=0 -n codimd
 ```
 
 Next, annotate the `SnapshotGroup` with the timestamp of the snapshot you want.
 
 For example, here we'll use timestamp `1585945609`.
 ```bash
-$ kubectl get volumesnapshot -n notepad
+$ kubectl get volumesnapshot -n codimd
 NAME                           AGE
-hackmd-1585945609              15s
-hackmd-postgresql-1585945609   15s
+codimd-1585945609              15s
+codimd-postgresql-1585945609   15s
 ```
 
 ```bash
-kubectl annotate snapshotgroup/hackmd-postgresql -n notepad --overwrite \
+kubectl annotate snapshotgroup/codimd-postgresql -n codimd --overwrite \
   "gemini.fairwinds.com/restore=1585945609"
 ```
 
@@ -123,16 +105,18 @@ This will:
 Note: If your PVC gets stuck in `Terminating`, this might be related to rate-limiting from the DO API (check [this issue](https://github.com/FairwindsOps/gemini/issues/29) for more info) You can force destroy the PVC by running:
 
 ```bash
-kubectl -n notepad patch pvc data-hackmd-postgresql-0 -p '{"metadata":{"finalizers": []}}' --type=merge
+kubectl -n codimd patch pvc data-codimd-postgresql-0 -p '{"metadata":{"finalizers": []}}' --type=merge
 ```
 
+Wait until you see that your PVC is back in `Bound` state.
+
 Finally, we can scale back up:
 ```bash
-kubectl scale all --all --replicas=1 -n notepad
+kubectl scale all --all --replicas=1 -n codimd
 ```
 
 ### Verify the restore
 ```bash
-kubectl port-forward svc/hackmd 3000:3000 -n notepad
+kubectl port-forward svc/codimd 3000:80 -n codimd
 ```
 Go back to your document. The second round of edits you made should be gone!

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
 	github.com/stretchr/testify v1.8.1
 	golang.org/x/time v0.3.0
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.1
 	k8s.io/apiextensions-apiserver v0.26.1
 	k8s.io/apimachinery v0.26.1
@@ -46,7 +47,6 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230217203603-ff9a8e8fa21d // indirect
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect

pkg/controller/controller.go

@@ -28,8 +28,8 @@ import (
 
 	"github.com/fairwindsops/gemini/pkg/kube"
 	"github.com/fairwindsops/gemini/pkg/snapshots"
-	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1"
-	listers "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/listers/snapshotgroup/v1beta1"
+	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1"
+	listers "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/listers/snapshotgroup/v1"
 )
 
 const defaultSnapshotReadyTimeoutSeconds = 60

pkg/controller/controller_test.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/fairwindsops/gemini/pkg/kube"
 	"github.com/fairwindsops/gemini/pkg/snapshots"
-	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1"
+	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1"
 )
 
 func newSnapshotGroup(name, namespace string) *snapshotgroup.SnapshotGroup {

pkg/kube/client.go

@@ -15,6 +15,7 @@
 package kube
 
 import (
+	"os"
 	"context"
 	"errors"
 	"time"
@@ -31,11 +32,11 @@ import (
 	"k8s.io/client-go/restmapper"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
-	snapshotgroupv1 "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1"
-	snapshotGroupClientset "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/clientset/versioned"
-	snapshotgroupInterface "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/clientset/versioned/typed/snapshotgroup/v1beta1"
-	"github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/informers/externalversions"
-	informers "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/informers/externalversions/snapshotgroup/v1beta1"
+	snapshotgroupv1 "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1"
+	snapshotGroupClientset "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/clientset/versioned"
+	snapshotgroupInterface "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/clientset/versioned/typed/snapshotgroup/v1"
+	"github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/informers/externalversions"
+	informers "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/informers/externalversions/snapshotgroup/v1"
 )
 
 const (
@@ -51,7 +52,7 @@ type Client struct {
 	Informer              informers.SnapshotGroupInformer
 	InformerFactory       externalversions.SharedInformerFactory
 	SnapshotClient        dynamic.NamespaceableResourceInterface
-	SnapshotGroupClient   snapshotgroupInterface.SnapshotgroupV1beta1Interface
+	SnapshotGroupClient   snapshotgroupInterface.SnapshotgroupV1Interface
 	VolumeSnapshotVersion string
 }
 
@@ -84,7 +85,7 @@ func createClient() *Client {
 	}
 
 	informerFactory := externalversions.NewSharedInformerFactory(sgClientSet, time.Second*30)
-	informer := informerFactory.Snapshotgroup().V1beta1().SnapshotGroups()
+	informer := informerFactory.Snapshotgroup().V1().SnapshotGroups()
 
 	resources, err := restmapper.GetAPIGroupResources(k8s.Discovery())
 	if err != nil {
@@ -113,15 +114,17 @@ func createClient() *Client {
 	}
 	snapshotClient := dynamicInterface.Resource(vsMapping.Resource)
 
-	if _, err = snapshotgroupv1.CreateCustomResourceDefinition("crd-ns", extClientSet); err != nil {
-		panic(err)
+	if os.Getenv("GEMINI_CREATE_CRD") != "" {
+		if _, err = snapshotgroupv1.CreateCustomResourceDefinition("crd-ns", extClientSet); err != nil {
+			panic(err)
+		}
 	}
 	return &Client{
 		K8s:                   k8s,
 		Informer:              informer,
 		InformerFactory:       informerFactory,
 		SnapshotClient:        snapshotClient,
-		SnapshotGroupClient:   sgClientSet.SnapshotgroupV1beta1(),
+		SnapshotGroupClient:   sgClientSet.SnapshotgroupV1(),
 		VolumeSnapshotVersion: VolumeSnapshotGroupName + "/" + volumeSnapshotVersion,
 	}
 }

pkg/kube/fake_client.go

@@ -23,8 +23,8 @@ import (
 	dynamicFake "k8s.io/client-go/dynamic/fake"
 	k8sfake "k8s.io/client-go/kubernetes/fake"
 
-	snapshotGroupsFake "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/clientset/versioned/fake"
-	snapshotGroupExternalVersions "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1/apis/informers/externalversions"
+	snapshotGroupsFake "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/clientset/versioned/fake"
+	snapshotGroupExternalVersions "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1/apis/informers/externalversions"
 )
 
 var noResync = func() time.Duration { return 0 }
@@ -42,11 +42,11 @@ func createFakeClient() *Client {
 
 	snapshotGroupClientSet := snapshotGroupsFake.NewSimpleClientset(objects...)
 	informerFactory := snapshotGroupExternalVersions.NewSharedInformerFactory(snapshotGroupClientSet, noResync())
-	informer := informerFactory.Snapshotgroup().V1beta1().SnapshotGroups()
+	informer := informerFactory.Snapshotgroup().V1().SnapshotGroups()
 
 	volumeSnapshotVersionResource := schema.GroupVersionResource{
 		Group:    VolumeSnapshotGroupName,
-		Version:  "v1beta1",
+		Version:  "v1",
 		Resource: VolumeSnapshotKind,
 	}
 	dynamic := dynamicFake.NewSimpleDynamicClientWithCustomListKinds(k8sruntime.NewScheme(), map[schema.GroupVersionResource]string{
@@ -59,6 +59,6 @@ func createFakeClient() *Client {
 		Informer:            informer,
 		InformerFactory:     informerFactory,
 		SnapshotClient:      snapshotClient,
-		SnapshotGroupClient: snapshotGroupClientSet.SnapshotgroupV1beta1(),
+		SnapshotGroupClient: snapshotGroupClientSet.SnapshotgroupV1(),
 	}
 }

pkg/snapshots/groups.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/fairwindsops/gemini/pkg/kube"
-	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1"
+	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1"
 )
 
 func updateSnapshotGroup(sg *snapshotgroup.SnapshotGroup) error {

pkg/snapshots/pvc.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/fairwindsops/gemini/pkg/kube"
-	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1beta1"
+	snapshotgroup "github.com/fairwindsops/gemini/pkg/types/snapshotgroup/v1"
 )
 
 func getPVCName(sg *snapshotgroup.SnapshotGroup) string {