Skip to content

Commit ea5a726

Browse files
d8660091d8660091
committed
Group docker simple flow tests to suite
1 parent b399c50 commit ea5a726

File tree

7 files changed

+126
-59
lines changed

7 files changed

+126
-59
lines changed

‎cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -224,8 +224,9 @@ phases:
224224
-i ${INTEGRATION_TEST_INSTANCE_PROFILE}
225225
-m ${INTEGRATION_TEST_MAX_EC2_COUNT}
226226
-p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
227-
-r ${TESTS}
227+
-r 'Test'
228228
-v 4
229+
--select ${TESTS}
229230
--skip ${SKIPPED_TESTS}
230231
--bundles-override=${BUNDLES_OVERRIDE}
231232
--cleanup-resources=true

‎cmd/integration_test/cmd/run.go

+4
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const (
2020
regexFlagName = "regex"
2121
maxInstancesFlagName = "max-instances"
2222
maxConcurrentTestsFlagName = "max-concurrent-tests"
23+
selectFlagName = "select"
2324
skipFlagName = "skip"
2425
bundlesOverrideFlagName = "bundles-override"
2526
cleanupResourcesFlagName = "cleanup-resources"
@@ -64,6 +65,7 @@ func init() {
6465
runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run")
6566
runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count")
6667
runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time")
68+
runE2ECmd.Flags().StringSlice(selectFlagName, nil, "List of tests to select")
6769
runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip")
6870
runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override")
6971
runE2ECmd.Flags().Bool(cleanupResourcesFlagName, false, "Flag to indicate if test resources should be cleaned up automatically as tests complete")
@@ -86,6 +88,7 @@ func runE2E(ctx context.Context) error {
8688
testRegex := viper.GetString(regexFlagName)
8789
maxInstances := viper.GetInt(maxInstancesFlagName)
8890
maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName)
91+
testsToSelect := viper.GetStringSlice(selectFlagName)
8992
testsToSkip := viper.GetStringSlice(skipFlagName)
9093
bundlesOverride := viper.GetBool(bundlesOverrideFlagName)
9194
cleanupResources := viper.GetBool(cleanupResourcesFlagName)
@@ -100,6 +103,7 @@ func runE2E(ctx context.Context) error {
100103
StorageBucket: storageBucket,
101104
JobId: jobId,
102105
Regex: testRegex,
106+
TestsToSelect: testsToSelect,
103107
TestsToSkip: testsToSkip,
104108
BundlesOverride: bundlesOverride,
105109
CleanupResources: cleanupResources,

‎internal/test/e2e/artifacts.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package e2e
33
import (
44
"fmt"
55
"path/filepath"
6+
"strings"
67

78
"github.com/go-logr/logr"
89

@@ -58,7 +59,8 @@ func (e *E2ESession) uploadJUnitReportFromInstance(testName string) {
5859
func (e *E2ESession) downloadJUnitReportToLocalDisk(testName, destinationFolder string) {
5960
junitFile := "junit-testing.xml"
6061
key := filepath.Join(e.generatedArtifactsPath(), testName, junitFile)
61-
dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", testName))
62+
// Subtest contains '/' in the name.
63+
dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", strings.Replace(testName, "/", "-", -1)))
6264

6365
e.logger.V(1).Info("Downloading JUnit report to disk", "dst", dst)
6466
if err := s3.DownloadToDisk(e.session, key, e.storageBucket, dst); err != nil {

‎internal/test/e2e/list.go

+51-8
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,21 @@ package e2e
22

33
import (
44
"bufio"
5+
"bytes"
56
"context"
67
"fmt"
78
"path/filepath"
9+
"regexp"
810
"strings"
911

1012
"github.com/aws/eks-anywhere/pkg/executables"
1113
"github.com/aws/eks-anywhere/pkg/types"
14+
e2etest "github.com/aws/eks-anywhere/test/e2e"
1215
)
1316

14-
func listTests(regex string, testsToSkip []string) (tests, skippedTests []string, err error) {
17+
// regex will filter out the tests that match the regex, but it does't see subtests.
18+
// TestsToSelect and testsToSkip will filter out all test cases including subtests.
19+
func listTests(regex string, testsToSelect []string, testsToSkip []string) (tests, skippedTests []string, err error) {
1520
e := executables.NewExecutable(filepath.Join("bin", e2eBinary))
1621
ctx := context.Background()
1722
testResponse, err := e.Execute(ctx, "-test.list", regex)
@@ -21,16 +26,26 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string
2126

2227
skipLookup := types.SliceToLookup(testsToSkip)
2328
scanner := bufio.NewScanner(&testResponse)
24-
for scanner.Scan() {
25-
line := scanner.Text()
26-
if skipLookup.IsPresent(line) {
27-
skippedTests = append(skippedTests, line)
29+
rawTestList := processTestListOutput(testResponse)
30+
31+
for _, t := range rawTestList {
32+
selected := len(testsToSelect) == 0
33+
for _, s := range testsToSelect {
34+
re := regexp.MustCompile(s)
35+
if re.MatchString(t) {
36+
selected = true
37+
break
38+
}
39+
}
40+
if !selected {
2841
continue
2942
}
30-
31-
if strings.HasPrefix(line, "Test") {
32-
tests = append(tests, line)
43+
if skipLookup.IsPresent(t) {
44+
skippedTests = append(skippedTests, t)
45+
continue
3346
}
47+
48+
tests = append(tests, t)
3449
}
3550

3651
if err := scanner.Err(); err != nil {
@@ -39,3 +54,31 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string
3954

4055
return tests, skippedTests, nil
4156
}
57+
58+
// parse the output of -test.list, and add subtests
59+
func processTestListOutput(b bytes.Buffer) (tests []string) {
60+
s := bufio.NewScanner(&b)
61+
for s.Scan() {
62+
line := s.Text()
63+
if !strings.HasPrefix(line, "Test") {
64+
continue
65+
}
66+
67+
if !strings.HasSuffix(line, "Suite") {
68+
tests = append(tests, line)
69+
continue
70+
}
71+
72+
if strings.HasSuffix(line, "Suite") {
73+
for k, s := range e2etest.Suites {
74+
if strings.HasSuffix(line, k) {
75+
for _, st := range s {
76+
tests = append(tests, line+"/"+st.GetName())
77+
}
78+
break
79+
}
80+
}
81+
}
82+
}
83+
return tests
84+
}

‎internal/test/e2e/run.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type ParallelRunConf struct {
4545
StorageBucket string
4646
JobId string
4747
Regex string
48+
TestsToSelect []string
4849
TestsToSkip []string
4950
BundlesOverride bool
5051
CleanupResources bool
@@ -65,7 +66,7 @@ type (
6566

6667
// RunTestsInParallel Run Tests in parallel by spawning multiple admin machines.
6768
func RunTestsInParallel(conf ParallelRunConf) error {
68-
testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSkip)
69+
testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSelect, conf.TestsToSkip)
6970
if err != nil {
7071
return err
7172
}
@@ -282,15 +283,16 @@ func RunTests(conf instanceRunConf, inventoryCatalogue map[string]*hardwareCatal
282283
}
283284

284285
func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResult, err error) {
285-
e.logger.V(1).Info("Running e2e tests", "regex", regex)
286286
command := "GOVERSION=go1.16.6 gotestsum --junitfile=junit-testing.xml --raw-command --format=standard-verbose --hide-summary=all --ignore-non-json-output-lines -- test2json -t -p e2e ./bin/e2e.test -test.v"
287287

288288
if regex != "" {
289-
command = fmt.Sprintf("%s -test.run \"^(%s)$\" -test.timeout %s", command, regex, e2eTimeout)
289+
command = fmt.Sprintf("%s -test.run \"^%s$\" -test.timeout %s", command, regex, e2eTimeout)
290290
}
291291

292292
command = e.commandWithEnvVars(command)
293293

294+
e.logger.V(1).Info("Running e2e tests", "regex", regex, "command", command)
295+
294296
opt := ssm.WithOutputToCloudwatch()
295297

296298
testCommandResult, err = ssm.RunCommand(

‎test/e2e/docker_test.go

+13-46
Original file line numberDiff line numberDiff line change
@@ -621,52 +621,6 @@ func TestDockerKubernetes130RegistryMirrorInsecureSkipVerify(t *testing.T) {
621621
runRegistryMirrorConfigFlow(test)
622622
}
623623

624-
// Simple flow
625-
func TestDockerKubernetes126SimpleFlow(t *testing.T) {
626-
test := framework.NewClusterE2ETest(
627-
t,
628-
framework.NewDocker(t),
629-
framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)),
630-
)
631-
runSimpleFlow(test)
632-
}
633-
634-
func TestDockerKubernetes127SimpleFlow(t *testing.T) {
635-
test := framework.NewClusterE2ETest(
636-
t,
637-
framework.NewDocker(t),
638-
framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)),
639-
)
640-
runSimpleFlow(test)
641-
}
642-
643-
func TestDockerKubernetes128SimpleFlow(t *testing.T) {
644-
test := framework.NewClusterE2ETest(
645-
t,
646-
framework.NewDocker(t),
647-
framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)),
648-
)
649-
runSimpleFlow(test)
650-
}
651-
652-
func TestDockerKubernetes129SimpleFlow(t *testing.T) {
653-
test := framework.NewClusterE2ETest(
654-
t,
655-
framework.NewDocker(t),
656-
framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
657-
)
658-
runSimpleFlow(test)
659-
}
660-
661-
func TestDockerKubernetes130SimpleFlow(t *testing.T) {
662-
test := framework.NewClusterE2ETest(
663-
t,
664-
framework.NewDocker(t),
665-
framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
666-
)
667-
runSimpleFlow(test)
668-
}
669-
670624
// Stacked etcd
671625
func TestDockerKubernetesStackedEtcd(t *testing.T) {
672626
test := framework.NewClusterE2ETest(t,
@@ -1480,3 +1434,16 @@ func TestDockerKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
14801434
)
14811435
runKubeletConfigurationFlow(test)
14821436
}
1437+
1438+
func TestDockerKubernetesSimpleFlowSuite(t *testing.T) {
1439+
for _, ts := range Suites[simpleFlowSubtest.GetSuiteSuffix()] {
1440+
t.Run(ts.GetName(), func(t *testing.T) {
1441+
test := framework.NewClusterE2ETest(
1442+
t,
1443+
framework.NewDocker(t),
1444+
framework.WithClusterFiller(api.WithKubernetesVersion(ts.(*SimpleFlowSubtest).KubeVersion)),
1445+
)
1446+
runSimpleFlow(test)
1447+
})
1448+
}
1449+
}

‎test/e2e/test_suite.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package e2e
2+
3+
import (
4+
"strings"
5+
6+
"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
7+
)
8+
9+
var kubeVersions = []v1alpha1.KubernetesVersion{v1alpha1.Kube126, v1alpha1.Kube127, v1alpha1.Kube128, v1alpha1.Kube129, v1alpha1.Kube130}
10+
11+
// Subtest is an interface to represent a test case.
12+
type Subtest interface {
13+
// Return the name of the subtest case.
14+
GetName() string
15+
// Return the name suffix of the parent test, which includes this subtest. It should be "*Suite".
16+
GetSuiteSuffix() string
17+
}
18+
19+
// Suites contain all test suites. The key is the suite suffix.
20+
var Suites = map[string][]Subtest{
21+
simpleFlowSubtest.GetSuiteSuffix(): {},
22+
}
23+
24+
// SimpleFlowSubtest is a struct to represent a simple flow test.
25+
type SimpleFlowSubtest struct {
26+
KubeVersion v1alpha1.KubernetesVersion
27+
}
28+
29+
// GetName returns the subtest name.
30+
func (st *SimpleFlowSubtest) GetName() string {
31+
return strings.ReplaceAll(string(st.KubeVersion), ".", "")
32+
}
33+
34+
// GetSuiteSuffix returns the Suite suffix.
35+
func (st *SimpleFlowSubtest) GetSuiteSuffix() string {
36+
return "SimpleFlowSuite"
37+
}
38+
39+
// Make sure the SimpleFlowSubtest implements the Subtest interface.
40+
var simpleFlowSubtest Subtest = (*SimpleFlowSubtest)(nil)
41+
42+
// Init SimpleFlowSuite.
43+
func init() {
44+
s := simpleFlowSubtest.GetSuiteSuffix()
45+
for _, k := range kubeVersions {
46+
Suites[s] = append(Suites[s], &SimpleFlowSubtest{KubeVersion: k})
47+
}
48+
}

0 commit comments

Comments
 (0)