Skip to content

Commit 3e9c9a9

Browse files
committed
Add command to cleanup tinkerbell test resources
1 parent 46e3955 commit 3e9c9a9

File tree

14 files changed

+579
-179
lines changed

14 files changed

+579
-179
lines changed

Makefile

+3-1
Original file line numberDiff line numberDiff line change
@@ -683,8 +683,10 @@ e2e-tests-binary:
683683
GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) test ./test/e2e -c -o "$(E2E_OUTPUT_FILE)" -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)"
684684

685685
.PHONY: build-integration-test-binary
686+
build-integration-test-binary: ALL_LINKER_FLAGS := $(LINKER_FLAGS) -X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL) -s -w -buildid='' -extldflags -static
687+
build-integration-test-binary: LINKER_FLAGS_ARG := -ldflags "$(ALL_LINKER_FLAGS)"
686688
build-integration-test-binary:
687-
GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
689+
GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build $(LINKER_FLAGS_ARG) -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
688690

689691
.PHONY: conformance
690692
conformance:

cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml

+6
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,12 @@ phases:
203203
--insecure
204204
--ignoreErrors
205205
-v 4
206+
- >
207+
./bin/test e2e cleanup tinkerbell
208+
--storage-bucket ${INTEGRATION_TEST_STORAGE_BUCKET}
209+
--instance-config ${INTEGRATION_TEST_INFRA_CONFIG}
210+
--dry-run
211+
-v 4
206212
build:
207213
commands:
208214
- export JOB_ID=$CODEBUILD_BUILD_ID
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package cmd
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
8+
"github.com/aws/aws-sdk-go/aws/session"
9+
"github.com/spf13/cobra"
10+
11+
"github.com/aws/eks-anywhere/internal/pkg/ssm"
12+
"github.com/aws/eks-anywhere/internal/test/cleanup"
13+
"github.com/aws/eks-anywhere/internal/test/e2e"
14+
"github.com/aws/eks-anywhere/pkg/dependencies"
15+
"github.com/aws/eks-anywhere/pkg/errors"
16+
"github.com/aws/eks-anywhere/pkg/executables"
17+
"github.com/aws/eks-anywhere/pkg/logger"
18+
"github.com/aws/eks-anywhere/pkg/providers/tinkerbell/hardware"
19+
)
20+
21+
var cleanUpTinkerbellCmd = &cobra.Command{
22+
Use: "tinkerbell",
23+
Short: "Clean up tinkerbell e2e resources",
24+
Long: "Deletes vms created for e2e testing on vsphere and powers off metal machines",
25+
SilenceUsage: true,
26+
PreRun: preRunCleanUpNutanixSetup,
27+
RunE: func(cmd *cobra.Command, _ []string) error {
28+
return cleanUpTinkerbellTestResources(cmd.Context())
29+
},
30+
}
31+
32+
var (
33+
storageBucket string
34+
instanceConfig string
35+
dryRun bool
36+
)
37+
38+
func init() {
39+
cleanUpInstancesCmd.AddCommand(cleanUpTinkerbellCmd)
40+
cleanUpTinkerbellCmd.Flags().StringVarP(&storageBucket, storageBucketFlagName, "s", "", "S3 bucket name where tinkerbell hardware inventory files are stored")
41+
cleanUpTinkerbellCmd.Flags().StringVar(&instanceConfig, instanceConfigFlagName, "", "File path to the instance-config.yml config")
42+
cleanUpTinkerbellCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Run command without deleting or powering off any resources")
43+
44+
if err := cleanUpTinkerbellCmd.MarkFlagRequired(storageBucketFlagName); err != nil {
45+
log.Fatalf("Error marking flag %s as required: %v", storageBucketFlagName, err)
46+
}
47+
48+
if err := cleanUpTinkerbellCmd.MarkFlagRequired(instanceConfigFlagName); err != nil {
49+
log.Fatalf("Error marking flag %s as required: %v", instanceConfigFlagName, err)
50+
}
51+
}
52+
53+
// cleanUpTinkerbellTestResources deletes any test runner vm in vsphere and powers off all metal machines.
54+
func cleanUpTinkerbellTestResources(ctx context.Context) error {
55+
session, err := session.NewSession()
56+
if err != nil {
57+
return fmt.Errorf("creating session: %w", err)
58+
}
59+
60+
deps, err := dependencies.NewFactory().WithGovc().Build(ctx)
61+
if err != nil {
62+
return err
63+
}
64+
defer deps.Close(ctx)
65+
govc := deps.Govc
66+
67+
infraConfig, err := e2e.ReadRunnerConfig(instanceConfig)
68+
if err != nil {
69+
return fmt.Errorf("reading vms config for tests: %v", err)
70+
}
71+
72+
govc.Configure(
73+
executables.GovcConfig{
74+
Username: infraConfig.Username,
75+
Password: infraConfig.Password,
76+
URL: infraConfig.URL,
77+
Insecure: infraConfig.Insecure,
78+
Datacenter: infraConfig.Datacenter,
79+
},
80+
)
81+
82+
var errs []error
83+
84+
if err := deleteSSMInstances(ctx, session); len(err) != 0 {
85+
errs = append(errs, err...)
86+
}
87+
88+
if err := deleteRunners(ctx, govc, infraConfig.Folder); len(err) != 0 {
89+
errs = append(errs, err...)
90+
}
91+
92+
if err := powerOffMachines(ctx, session); len(err) != 0 {
93+
errs = append(errs, err...)
94+
}
95+
96+
return errors.NewAggregate(errs)
97+
}
98+
99+
func deleteSSMInstances(ctx context.Context, session *session.Session) []error {
100+
var errs []error
101+
if ssmInstances, err := e2e.ListTinkerbellSSMInstances(ctx, session); err != nil {
102+
errs = append(errs, fmt.Errorf("listing ssm instances: %w", err))
103+
} else if dryRun {
104+
logger.Info("Found SSM instances", "instanceIDs", ssmInstances.InstanceIDs, "activationIDs", ssmInstances.ActivationIDs)
105+
} else {
106+
if _, err := ssm.DeregisterInstances(session, ssmInstances.InstanceIDs...); err != nil {
107+
errs = append(errs, fmt.Errorf("deleting ssm instances: %w", err))
108+
}
109+
if _, err := ssm.DeleteActivations(session, ssmInstances.ActivationIDs...); err != nil {
110+
errs = append(errs, fmt.Errorf("deleting ssm activations: %w", err))
111+
}
112+
}
113+
114+
return errs
115+
}
116+
117+
func deleteRunners(ctx context.Context, govc *executables.Govc, folder string) []error {
118+
var errs []error
119+
if runners, err := govc.ListVMs(ctx, folder); err != nil {
120+
errs = append(errs, fmt.Errorf("listing tinkerbell runners: %w", err))
121+
} else if dryRun {
122+
logger.Info("Found VM Runners", "vms", runners)
123+
} else {
124+
for _, vm := range runners {
125+
if err := govc.DeleteVM(ctx, vm.Path); err != nil {
126+
errs = append(errs, fmt.Errorf("deleting tinkerbell runner %s: %w", vm, err))
127+
}
128+
}
129+
}
130+
131+
return errs
132+
}
133+
134+
func powerOffMachines(_ context.Context, session *session.Session) []error {
135+
var errs []error
136+
if machines, err := e2e.ReadTinkerbellMachinePool(session, storageBucket); err != nil {
137+
errs = append(errs, fmt.Errorf("reading tinkerbell machine pool: %v", err))
138+
} else if dryRun {
139+
logger.Info("Metal machine pool", "machines", names(machines))
140+
} else {
141+
if err = cleanup.PowerOffTinkerbellMachines(machines, true); err != nil {
142+
errs = append(errs, fmt.Errorf("powering off tinkerbell machines: %v", err))
143+
}
144+
}
145+
146+
return errs
147+
}
148+
149+
func names(h []*hardware.Machine) []string {
150+
names := make([]string, 0, len(h))
151+
for _, m := range h {
152+
names = append(names, m.Hostname)
153+
}
154+
155+
return names
156+
}

internal/pkg/ssm/activation.go

+28-9
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package ssm
33
import (
44
"fmt"
55

6+
"github.com/aws/aws-sdk-go/aws"
67
"github.com/aws/aws-sdk-go/aws/session"
78
"github.com/aws/aws-sdk-go/service/ssm"
89
)
@@ -12,7 +13,14 @@ type ActivationInfo struct {
1213
ActivationID string
1314
}
1415

15-
func CreateActivation(session *session.Session, instanceName, role string) (*ActivationInfo, error) {
16+
// Tag is an SSM tag.
17+
type Tag struct {
18+
Key string
19+
Value string
20+
}
21+
22+
// CreateActivation creates an SSM Hybrid activation.
23+
func CreateActivation(session *session.Session, instanceName, role string, tags ...Tag) (*ActivationInfo, error) {
1624
s := ssm.New(session)
1725

1826
request := ssm.CreateActivationInput{
@@ -21,6 +29,12 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act
2129
IamRole: &role,
2230
}
2331

32+
for _, tag := range tags {
33+
request.Tags = append(request.Tags,
34+
&ssm.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)},
35+
)
36+
}
37+
2438
result, err := s.CreateActivation(&request)
2539
if err != nil {
2640
return nil, fmt.Errorf("failed to activate ssm instance %s: %v", instanceName, err)
@@ -29,17 +43,22 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act
2943
return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil
3044
}
3145

32-
func DeleteActivation(session *session.Session, activationId string) (*ssm.DeleteActivationOutput, error) {
46+
// DeleteActivations deletes SSM activations.
47+
func DeleteActivations(session *session.Session, ids ...string) ([]*ssm.DeleteActivationOutput, error) {
3348
s := ssm.New(session)
49+
var outputs []*ssm.DeleteActivationOutput
50+
for _, id := range ids {
51+
request := ssm.DeleteActivationInput{
52+
ActivationId: &id,
53+
}
3454

35-
request := ssm.DeleteActivationInput{
36-
ActivationId: &activationId,
37-
}
55+
result, err := s.DeleteActivation(&request)
56+
if err != nil {
57+
return nil, fmt.Errorf("failed to delete ssm activation: %v", err)
58+
}
3859

39-
result, err := s.DeleteActivation(&request)
40-
if err != nil {
41-
return nil, fmt.Errorf("failed to delete ssm activation: %v", err)
60+
outputs = append(outputs, result)
4261
}
4362

44-
return result, nil
63+
return outputs, nil
4564
}

internal/pkg/ssm/instance.go

+35-6
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package ssm
22

33
import (
4+
"context"
45
"fmt"
56

7+
"github.com/aws/aws-sdk-go/aws"
68
"github.com/aws/aws-sdk-go/aws/session"
79
"github.com/aws/aws-sdk-go/service/ssm"
810
)
@@ -30,16 +32,43 @@ func GetInstanceByActivationId(session *session.Session, id string) (*ssm.Instan
3032
return infoList[0], nil
3133
}
3234

33-
func DeregisterInstance(session *session.Session, id string) (*ssm.DeregisterManagedInstanceOutput, error) {
35+
// DeregisterInstances deregisters SSM instances.
36+
func DeregisterInstances(session *session.Session, ids ...string) ([]*ssm.DeregisterManagedInstanceOutput, error) {
3437
s := ssm.New(session)
35-
input := ssm.DeregisterManagedInstanceInput{
36-
InstanceId: &id,
38+
var outputs []*ssm.DeregisterManagedInstanceOutput
39+
for _, id := range ids {
40+
input := ssm.DeregisterManagedInstanceInput{
41+
InstanceId: &id,
42+
}
43+
44+
output, err := s.DeregisterManagedInstance(&input)
45+
if err != nil {
46+
return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err)
47+
}
48+
49+
outputs = append(outputs, output)
3750
}
3851

39-
output, err := s.DeregisterManagedInstance(&input)
52+
return outputs, nil
53+
}
54+
55+
func ListInstancesByTags(ctx context.Context, session *session.Session, tags ...Tag) ([]*ssm.InstanceInformation, error) {
56+
s := ssm.New(session)
57+
input := ssm.DescribeInstanceInformationInput{
58+
Filters: make([]*ssm.InstanceInformationStringFilter, 0, len(tags)),
59+
}
60+
61+
for _, tag := range tags {
62+
input.Filters = append(input.Filters, &ssm.InstanceInformationStringFilter{
63+
Key: aws.String("tag:" + tag.Key),
64+
Values: aws.StringSlice([]string{tag.Value}),
65+
})
66+
}
67+
68+
output, err := s.DescribeInstanceInformation(&input)
4069
if err != nil {
41-
return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err)
70+
return nil, fmt.Errorf("listing ssm instances by tags: %v", err)
4271
}
4372

44-
return output, nil
73+
return output.InstanceInformationList, nil
4574
}

0 commit comments

Comments
 (0)