Skip to content

Commit e85ca3e

Browse files
committedJun 17, 2024·
allow KinD clusters to be deleted or retained
Modifies the KinD Fixture to make it more stable and more flexible. The default behaviour is for the KinD Fixture to delete the KinD cluster only if the KinD cluster existed before the Fixture is started. Similarly, if the KinD Cluster did *not* exist before the Fixture is started, the Fixture deletes the KinD Cluster when it is stopped. This commit adds a WithDeleteOnStop() and WithRetainOnStop() modifier to the KinD Fixture creation allowing users to indicate if they want to change that default behaviour. In addition to the above enhancements, this commit makes our testing more stable by adding some KinD cluster cleanup to our Makefile as well as serializing execution of longer-running, resource-intensive tests. Issue #13 Signed-off-by: Jay Pipes <jaypipes@gmail.com>
1 parent 5e64948 commit e85ca3e

File tree

9 files changed

+231
-70
lines changed

9 files changed

+231
-70
lines changed
 

‎.github/workflows/gate-tests.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ jobs:
3838
- name: run non-Kind tests
3939
env:
4040
SKIP_KIND: 1
41-
run: go test -v ./...
41+
run: make test
4242
test-all:
4343
strategy:
4444
matrix:
@@ -69,4 +69,4 @@ jobs:
6969
with:
7070
cluster_name: kind
7171
- name: run all tests
72-
run: go test -v ./...
72+
run: make test-all

‎Makefile

+36-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,44 @@
11
VERSION ?= $(shell git describe --tags --always --dirty)
2+
CLUSTERS = $(shell kind get clusters)
23

34
.PHONY: test
45

6+
kind-install:
7+
ifeq (, $(shell command -v kind))
8+
go install sigs.k8s.io/kind@v0.23.0
9+
endif
10+
511
# We clear the test cache because some of the tests require an out-of-band KinD
612
# cluster to run against and we want to re-run tests against that KinD cluster
713
# instead of from cached unit test results.
8-
test:
14+
clear-test-cache:
15+
@echo -n "clearing Go test cache ... "
916
@go clean -testcache
10-
@go test -v ./...
17+
@echo "ok."
18+
19+
kind-clear-clusters:
20+
ifneq (, $(shell command -v kind))
21+
ifneq (, $(CLUSTERS))
22+
@echo -n "clearing KinD clusters ... "
23+
@for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done
24+
@echo "ok."
25+
endif
26+
endif
27+
28+
kind-create-cluster:
29+
ifneq (, $(shell command -v kind))
30+
@echo -n "creating 'kind' cluster ... "
31+
@kind create cluster -q
32+
@echo "ok."
33+
@sleep 5
34+
endif
35+
36+
test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple
37+
38+
test-kind-simple:
39+
@go test -v ./parse_test.go
40+
@go test -v ./eval_test.go
41+
42+
test-all: test kind-clear-clusters
43+
@go test -v ./fixtures/kind/kind_test.go
44+
@go test -v ./placement_test.go

‎eval_test.go

+15-55
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,19 @@
55
package kube_test
66

77
import (
8-
"bufio"
9-
"bytes"
10-
"fmt"
11-
"os"
128
"path/filepath"
139
"testing"
1410

1511
"github.com/gdt-dev/gdt"
1612
gdtcontext "github.com/gdt-dev/gdt/context"
17-
kindfix "github.com/gdt-dev/kube/fixtures/kind"
1813
"github.com/stretchr/testify/require"
14+
15+
kindfix "github.com/gdt-dev/kube/fixtures/kind"
16+
"github.com/gdt-dev/kube/testutil"
1917
)
2018

2119
func TestListPodsEmpty(t *testing.T) {
22-
skipIfNoKind(t)
20+
testutil.SkipIfNoKind(t)
2321
require := require.New(t)
2422

2523
fp := filepath.Join("testdata", "list-pods-empty.yaml")
@@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) {
3634
}
3735

3836
func TestGetPodNotFound(t *testing.T) {
39-
skipIfNoKind(t)
37+
testutil.SkipIfNoKind(t)
4038
require := require.New(t)
4139

4240
fp := filepath.Join("testdata", "get-pod-not-found.yaml")
@@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) {
5351
}
5452

5553
func TestCreateUnknownResource(t *testing.T) {
56-
skipIfNoKind(t)
54+
testutil.SkipIfNoKind(t)
5755
require := require.New(t)
5856

5957
fp := filepath.Join("testdata", "create-unknown-resource.yaml")
@@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) {
7068
}
7169

7270
func TestDeleteResourceNotFound(t *testing.T) {
73-
skipIfNoKind(t)
71+
testutil.SkipIfNoKind(t)
7472
require := require.New(t)
7573

7674
fp := filepath.Join("testdata", "delete-resource-not-found.yaml")
@@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) {
8785
}
8886

8987
func TestDeleteUnknownResource(t *testing.T) {
90-
skipIfNoKind(t)
88+
testutil.SkipIfNoKind(t)
9189
require := require.New(t)
9290

9391
fp := filepath.Join("testdata", "delete-unknown-resource.yaml")
@@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) {
104102
}
105103

106104
func TestPodCreateGetDelete(t *testing.T) {
107-
skipIfNoKind(t)
105+
testutil.SkipIfNoKind(t)
108106
require := require.New(t)
109107

110108
fp := filepath.Join("testdata", "create-get-delete-pod.yaml")
@@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) {
121119
}
122120

123121
func TestMatches(t *testing.T) {
124-
skipIfNoKind(t)
122+
testutil.SkipIfNoKind(t)
125123
require := require.New(t)
126124

127125
fp := filepath.Join("testdata", "matches.yaml")
@@ -138,7 +136,7 @@ func TestMatches(t *testing.T) {
138136
}
139137

140138
func TestConditions(t *testing.T) {
141-
skipIfNoKind(t)
139+
testutil.SkipIfNoKind(t)
142140
require := require.New(t)
143141

144142
fp := filepath.Join("testdata", "conditions.yaml")
@@ -155,7 +153,7 @@ func TestConditions(t *testing.T) {
155153
}
156154

157155
func TestJSON(t *testing.T) {
158-
skipIfNoKind(t)
156+
testutil.SkipIfNoKind(t)
159157
require := require.New(t)
160158

161159
fp := filepath.Join("testdata", "json.yaml")
@@ -172,7 +170,7 @@ func TestJSON(t *testing.T) {
172170
}
173171

174172
func TestApply(t *testing.T) {
175-
skipIfNoKind(t)
173+
testutil.SkipIfNoKind(t)
176174
require := require.New(t)
177175

178176
fp := filepath.Join("testdata", "apply-deployment.yaml")
@@ -189,7 +187,7 @@ func TestApply(t *testing.T) {
189187
}
190188

191189
func TestEnvvarSubstitution(t *testing.T) {
192-
skipIfNoKind(t)
190+
testutil.SkipIfNoKind(t)
193191
require := require.New(t)
194192

195193
t.Setenv("pod_name", "foo")
@@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) {
208206
}
209207

210208
func TestWithLabels(t *testing.T) {
211-
skipIfNoKind(t)
209+
testutil.SkipIfNoKind(t)
212210
require := require.New(t)
213211

214212
fp := filepath.Join("testdata", "list-pods-with-labels.yaml")
@@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) {
223221
err = s.Run(ctx, t)
224222
require.Nil(err)
225223
}
226-
227-
func TestPlacementSpread(t *testing.T) {
228-
skipIfNoKind(t)
229-
require := require.New(t)
230-
231-
fp := filepath.Join("testdata", "placement-spread.yaml")
232-
233-
s, err := gdt.From(fp)
234-
require.Nil(err)
235-
require.NotNil(s)
236-
237-
kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml")
238-
239-
var b bytes.Buffer
240-
w := bufio.NewWriter(&b)
241-
ctx := gdtcontext.New(gdtcontext.WithDebug(w))
242-
243-
ctx = gdtcontext.RegisterFixture(
244-
ctx, "kind-three-workers-three-zones",
245-
kindfix.New(
246-
kindfix.WithClusterName("kind-three-workers-three-zones"),
247-
kindfix.WithConfigPath(kindCfgPath),
248-
),
249-
)
250-
251-
err = s.Run(ctx, t)
252-
require.Nil(err)
253-
254-
w.Flush()
255-
fmt.Println(b.String())
256-
}
257-
258-
func skipIfNoKind(t *testing.T) {
259-
_, found := os.LookupEnv("SKIP_KIND")
260-
if found {
261-
t.Skipf("skipping KinD-requiring test")
262-
}
263-
}

‎fixtures/kind/kind.go

+92-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55
package kind
66

77
import (
8+
"context"
89
"strings"
910

11+
gdtcontext "github.com/gdt-dev/gdt/context"
12+
"github.com/gdt-dev/gdt/debug"
1013
gdttypes "github.com/gdt-dev/gdt/types"
1114
"github.com/samber/lo"
1215
"sigs.k8s.io/kind/pkg/cluster"
@@ -20,9 +23,29 @@ import (
2023
type KindFixture struct {
2124
// provider is the KinD cluster provider
2225
provider *cluster.Provider
23-
// cfgStr contains the stringified KUBECONFIG that KinD returns in its
24-
// Provider.KubeConfig() call
25-
cfgStr string
26+
// deleteOnStop indicates that the KinD cluster should be deleted when
27+
// the fixture is stopped. Fixtures are stopped when test scenarios
28+
// utilizing the fixture have executed all their test steps.
29+
//
30+
// By default, KinD clusters that were already running when the fixture was
31+
// started are not deleted. This is to prevent the deletion of KinD
32+
// clusters that were in use outside of a gdt-kube execution. To override
33+
// this behaviour and always delete the KinD cluster on stop, use the
34+
// WithDeleteOnStop() modifier.
35+
deleteOnStop bool
36+
// retainOnStop indicates that the KinD cluster should *not* be deleted
37+
// when the fixture is stopped. Fixtures are stopped when test scenarios
38+
// utilizing the fixture have executed all their test steps.
39+
//
40+
// By default, KinD clusters that were *not* already running when the fixture was
41+
// started are deleted when the fixture stops. This is to clean up KinD
42+
// clusters that were created and used by the gdt-kube execution. To override
43+
// this behaviour and always retain the KinD cluster on stop, use the
44+
// WithRetainOnStop() modifier.
45+
retainOnStop bool
46+
// runningBeforeStart is true when the KinD cluster was already running
47+
// when the fixture was started.
48+
runningBeforeStart bool
2649
// ClusterName is the name of the KinD cluster. If not specified, gdt will
2750
// use the default cluster name that KinD uses, which is just "kind"
2851
ClusterName string
@@ -34,20 +57,35 @@ type KindFixture struct {
3457
ConfigPath string
3558
}
3659

37-
func (f *KindFixture) Start() {
60+
func (f *KindFixture) Start(ctx context.Context) {
61+
ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start")
62+
defer func() {
63+
ctx = gdtcontext.PopTrace(ctx)
64+
}()
3865
if f.ClusterName == "" {
3966
f.ClusterName = kindconst.DefaultClusterName
4067
}
4168
if f.isRunning() {
69+
debug.Println(ctx, "cluster %s already running", f.ClusterName)
70+
f.runningBeforeStart = true
4271
return
4372
}
4473
opts := []cluster.CreateOption{}
4574
if f.ConfigPath != "" {
75+
debug.Println(
76+
ctx, "using custom kind config %s for cluster %s",
77+
f.ConfigPath, f.ClusterName,
78+
)
4679
opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath))
4780
}
4881
if err := f.provider.Create(f.ClusterName, opts...); err != nil {
4982
panic(err)
5083
}
84+
debug.Println(ctx, "cluster %s successfully created", f.ClusterName)
85+
if !f.retainOnStop {
86+
f.deleteOnStop = true
87+
debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName)
88+
}
5189
}
5290

5391
func (f *KindFixture) isRunning() bool {
@@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool {
6199
return lo.Contains(clusterNames, f.ClusterName)
62100
}
63101

64-
func (f *KindFixture) Stop() {}
102+
func (f *KindFixture) Stop(ctx context.Context) {
103+
ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop")
104+
defer func() {
105+
ctx = gdtcontext.PopTrace(ctx)
106+
}()
107+
if !f.isRunning() {
108+
debug.Println(ctx, "cluster %s not running", f.ClusterName)
109+
return
110+
}
111+
if f.runningBeforeStart && !f.deleteOnStop {
112+
debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName)
113+
return
114+
}
115+
if f.deleteOnStop {
116+
if err := f.provider.Delete(f.ClusterName, ""); err != nil {
117+
panic(err)
118+
}
119+
debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName)
120+
}
121+
}
65122

66123
func (f *KindFixture) HasState(key string) bool {
67124
lkey := strings.ToLower(key)
@@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier {
119176
}
120177
}
121178

179+
// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when
180+
// the fixture stops. Fixtures are stopped when test scenarios utilizing the
181+
// fixture have executed all their test steps.
182+
//
183+
// By default, KinD clusters that were already running when the fixture was
184+
// started are not deleted. This is to prevent the deletion of KinD
185+
// clusters that were in use outside of a gdt-kube execution. To override
186+
// this behaviour and always delete the KinD cluster on stop, use the
187+
// WithDeleteOnStop() modifier.
188+
func WithDeleteOnStop() KindFixtureModifier {
189+
return func(f *KindFixture) {
190+
f.deleteOnStop = true
191+
}
192+
}
193+
194+
// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be
195+
// deleted when the fixture is stopped. Fixtures are stopped when test
196+
// scenarios utilizing the fixture have executed all their test steps.
197+
//
198+
// By default, KinD clusters that were *not* already running when the fixture
199+
// was started are deleted when the fixture stops. This is to clean up KinD
200+
// clusters that were created and used by the gdt-kube execution. To override
201+
// this behaviour and always retain the KinD cluster on stop, use the
202+
// WithRetainOnStop() modifier.
203+
func WithRetainOnStop() KindFixtureModifier {
204+
return func(f *KindFixture) {
205+
f.retainOnStop = true
206+
}
207+
}
208+
122209
// New returns a fixture that exposes Kubernetes configuration/context
123210
// information about a KinD cluster. If no such KinD cluster exists, one will
124211
// be created. If the KinD cluster is created, it is destroyed at the end of

‎fixtures/kind/kind_test.go

+19-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
package kind_test
66

77
import (
8+
"bufio"
9+
"bytes"
10+
"fmt"
811
"os"
912
"path/filepath"
1013
"testing"
@@ -25,10 +28,19 @@ func TestDefaultSingleControlPlane(t *testing.T) {
2528
require.Nil(err)
2629
require.NotNil(s)
2730

28-
ctx := gdtcontext.New()
29-
ctx = gdtcontext.RegisterFixture(ctx, "kind", kindfix.New())
31+
var b bytes.Buffer
32+
w := bufio.NewWriter(&b)
33+
ctx := gdtcontext.New(gdtcontext.WithDebug(w))
34+
ctx = gdtcontext.RegisterFixture(
35+
ctx, "kind",
36+
kindfix.New(
37+
kindfix.WithDeleteOnStop(),
38+
),
39+
)
3040

3141
err = s.Run(ctx, t)
42+
w.Flush()
43+
fmt.Println(b.String())
3244
require.Nil(err)
3345
}
3446

@@ -44,7 +56,9 @@ func TestOneControlPlaneOneWorker(t *testing.T) {
4456

4557
kindCfgPath := filepath.Join("testdata", "kind-config-one-cp-one-worker.yaml")
4658

47-
ctx := gdtcontext.New()
59+
var b bytes.Buffer
60+
w := bufio.NewWriter(&b)
61+
ctx := gdtcontext.New(gdtcontext.WithDebug(w))
4862
ctx = gdtcontext.RegisterFixture(
4963
ctx, "kind-one-cp-one-worker",
5064
kindfix.New(
@@ -54,6 +68,8 @@ func TestOneControlPlaneOneWorker(t *testing.T) {
5468
)
5569

5670
err = s.Run(ctx, t)
71+
w.Flush()
72+
fmt.Println(b.String())
5773
require.Nil(err)
5874
}
5975

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
require (
66
github.com/cenkalti/backoff/v4 v4.2.1
7-
github.com/gdt-dev/gdt v1.5.0
7+
github.com/gdt-dev/gdt v1.6.2
88
github.com/samber/lo v1.38.1
99
github.com/stretchr/testify v1.8.4
1010
gopkg.in/yaml.v3 v3.0.1

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
2121
github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
2222
github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro=
2323
github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
24-
github.com/gdt-dev/gdt v1.5.0 h1:HnLbiU8zXKK73SCGgm0R68dMvfry/J+pACHDODty48Y=
25-
github.com/gdt-dev/gdt v1.5.0/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU=
24+
github.com/gdt-dev/gdt v1.6.2 h1:ZHugSKIpMdO8hLQ1pGMPs0xGrLwQIutliDFfTjCkdys=
25+
github.com/gdt-dev/gdt v1.6.2/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU=
2626
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
2727
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
2828
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

‎placement_test.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Use and distribution licensed under the Apache license version 2.
2+
//
3+
// See the COPYING file in the root project directory for full text.
4+
5+
package kube_test
6+
7+
import (
8+
"bufio"
9+
"bytes"
10+
"fmt"
11+
"path/filepath"
12+
"testing"
13+
14+
"github.com/gdt-dev/gdt"
15+
gdtcontext "github.com/gdt-dev/gdt/context"
16+
"github.com/stretchr/testify/require"
17+
18+
kindfix "github.com/gdt-dev/kube/fixtures/kind"
19+
"github.com/gdt-dev/kube/testutil"
20+
)
21+
22+
func TestPlacementSpread(t *testing.T) {
23+
testutil.SkipIfNoKind(t)
24+
require := require.New(t)
25+
26+
fp := filepath.Join("testdata", "placement-spread.yaml")
27+
28+
s, err := gdt.From(fp)
29+
require.Nil(err)
30+
require.NotNil(s)
31+
32+
kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml")
33+
34+
var b bytes.Buffer
35+
w := bufio.NewWriter(&b)
36+
ctx := gdtcontext.New(gdtcontext.WithDebug(w))
37+
38+
ctx = gdtcontext.RegisterFixture(
39+
ctx, "kind-three-workers-three-zones",
40+
kindfix.New(
41+
kindfix.WithClusterName("kind-three-workers-three-zones"),
42+
kindfix.WithConfigPath(kindCfgPath),
43+
),
44+
)
45+
46+
err = s.Run(ctx, t)
47+
require.Nil(err)
48+
49+
w.Flush()
50+
fmt.Println(b.String())
51+
}

‎testutil/kind.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package testutil
2+
3+
import (
4+
"os"
5+
"testing"
6+
)
7+
8+
func SkipIfNoKind(t *testing.T) {
9+
_, found := os.LookupEnv("SKIP_KIND")
10+
if found {
11+
t.Skipf("skipping KinD-requiring test")
12+
}
13+
}

0 commit comments

Comments
 (0)
Please sign in to comment.