diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index c9b3a652a027..c5aee70b7e82 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -158,6 +158,71 @@ func (o *GinkgoRunSuiteOptions) SetIOStreams(streams genericclioptions.IOStreams o.IOStreams = streams } +// detectDuplicateTests creates test cases to report on duplicate test detection +// Flake for now. +func detectDuplicateTests(specs extensions.ExtensionTestSpecs) []*junitapi.JUnitTestCase { + const duplicateTestName = "There should not be duplicate tests" + + testOccurrences := make(map[string][]*extensions.ExtensionTestSpec) + + // Track all occurrences of each test name + for _, spec := range specs { + testOccurrences[spec.Name] = append(testOccurrences[spec.Name], spec) + } + + // Collect all duplicate tests + var duplicates []string + var allDuplicateDetails []string + + for testName, occurrences := range testOccurrences { + if len(occurrences) <= 1 { + continue // Not a duplicate + } + + duplicates = append(duplicates, testName) + logrus.Warnf("Duplicate test detected: %q appears %d times", testName, len(occurrences)) + + // Collect source information for all occurrences of this test + testDetails := fmt.Sprintf("\n=== Duplicate Test: %s ===", testName) + testDetails += fmt.Sprintf("\nFound %d occurrences:", len(occurrences)) + + for i, spec := range occurrences { + detail := fmt.Sprintf("\n Occurrence %d:", i+1) + detail += fmt.Sprintf("\n Source: %s", spec.ExtensionTestSpec.Source) + if len(spec.ExtensionTestSpec.CodeLocations) > 0 { + detail += fmt.Sprintf("\n CodeLocations: %s", strings.Join(spec.ExtensionTestSpec.CodeLocations, ", ")) + } + testDetails += detail + logrus.Warnf(" Occurrence %d: Source=%s", i+1, spec.ExtensionTestSpec.Source) + } + + allDuplicateDetails = append(allDuplicateDetails, testDetails) + } + + // Always create a success test case first + testCases := []*junitapi.JUnitTestCase{ + { + Name: duplicateTestName, + }, + } + + // If duplicates found, also create a failure case to make it a flake + if len(duplicates) > 0 { + totalDuplicates := len(duplicates) + logrus.Warnf("Found %d duplicate tests total", totalDuplicates) + fullDetails := strings.Join(allDuplicateDetails, "\n") + + testCases = append(testCases, &junitapi.JUnitTestCase{ + Name: duplicateTestName, + FailureOutput: &junitapi.FailureOutput{ + Output: fmt.Sprintf("Found %d duplicate tests:\n%s\n%s\n", totalDuplicates, strings.Join(duplicates, "\n"), fullDetails), + }, + }) + } + + return testCases +} + func max(a, b int) int { if a > b { return a @@ -243,6 +308,8 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc return fmt.Errorf("no tests to run") } + duplicateTestCases := detectDuplicateTests(specs) + k8sTestNames := map[string]bool{} for _, t := range specs { if strings.Contains(t.ExtensionTestSpec.Source, "hyperkube") { @@ -583,6 +650,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc var syntheticTestResults []*junitapi.JUnitTestCase var syntheticFailure bool syntheticTestResults = append(syntheticTestResults, stableClusterTestResults...) + syntheticTestResults = append(syntheticTestResults, duplicateTestCases...) timeSuffix := fmt.Sprintf("_%s", start.UTC().Format("20060102-150405")) diff --git a/pkg/test/ginkgo/cmd_runsuite_test.go b/pkg/test/ginkgo/cmd_runsuite_test.go new file mode 100644 index 000000000000..bccd4b6ea36b --- /dev/null +++ b/pkg/test/ginkgo/cmd_runsuite_test.go @@ -0,0 +1,195 @@ +package ginkgo + +import ( + "strings" + "testing" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + "github.com/openshift/origin/pkg/test/extensions" +) + +func Test_detectDuplicateTests(t *testing.T) { + tests := []struct { + name string + specs extensions.ExtensionTestSpecs + wantTestCaseCount int + wantSuccess bool + wantFailure bool + wantDuplicateCount int + wantFailureContains []string + }{ + { + name: "no duplicates", + specs: extensions.ExtensionTestSpecs{ + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-1", + Source: "source-1", + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-2", + Source: "source-2", + }, + }, + }, + wantTestCaseCount: 1, + wantSuccess: true, + wantFailure: false, + }, + { + name: "single duplicate test", + specs: extensions.ExtensionTestSpecs{ + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "duplicate-test", + Source: "source-1", + CodeLocations: []string{"file1.go:10"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "duplicate-test", + Source: "source-2", + CodeLocations: []string{"file2.go:20"}, + }, + }, + }, + wantTestCaseCount: 2, + wantSuccess: true, + wantFailure: true, + wantDuplicateCount: 1, + wantFailureContains: []string{ + "Found 1 duplicate tests", + "duplicate-test", + "=== Duplicate Test: duplicate-test ===", + "Found 2 occurrences:", + "Occurrence 1:", + "Source: source-1", + "CodeLocations: file1.go:10", + "Occurrence 2:", + "Source: source-2", + "CodeLocations: file2.go:20", + }, + }, + { + name: "multiple duplicate tests", + specs: extensions.ExtensionTestSpecs{ + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-a", + Source: "source-a1", + CodeLocations: []string{"file-a1.go:100"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-a", + Source: "source-a2", + CodeLocations: []string{"file-a2.go:200"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-b", + Source: "source-b1", + CodeLocations: []string{"file-b1.go:300"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-b", + Source: "source-b2", + CodeLocations: []string{"file-b2.go:400"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "test-b", + Source: "source-b3", + CodeLocations: []string{"file-b3.go:500"}, + }, + }, + { + ExtensionTestSpec: &extensiontests.ExtensionTestSpec{ + Name: "unique-test", + Source: "source-unique", + }, + }, + }, + wantTestCaseCount: 2, + wantSuccess: true, + wantFailure: true, + wantDuplicateCount: 2, + wantFailureContains: []string{ + "Found 2 duplicate tests", + "test-a", + "test-b", + "=== Duplicate Test: test-a ===", + "Found 2 occurrences:", + "=== Duplicate Test: test-b ===", + "Found 3 occurrences:", + "source-a1", + "source-a2", + "source-b1", + "source-b2", + "source-b3", + "file-a1.go:100", + "file-a2.go:200", + "file-b1.go:300", + "file-b2.go:400", + "file-b3.go:500", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testCases := detectDuplicateTests(tt.specs) + + // Check test case count + if len(testCases) != tt.wantTestCaseCount { + t.Errorf("detectDuplicateTests() returned %d test cases, want %d", len(testCases), tt.wantTestCaseCount) + } + + // Check for success test case + hasSuccess := false + hasFailure := false + var failureOutput string + + for _, tc := range testCases { + if tc.Name != "There should not be duplicate tests" { + t.Errorf("test case has wrong name: %q", tc.Name) + } + + if tc.FailureOutput == nil { + hasSuccess = true + } else { + hasFailure = true + failureOutput = tc.FailureOutput.Output + } + } + + if hasSuccess != tt.wantSuccess { + t.Errorf("detectDuplicateTests() hasSuccess = %v, want %v", hasSuccess, tt.wantSuccess) + } + + if hasFailure != tt.wantFailure { + t.Errorf("detectDuplicateTests() hasFailure = %v, want %v", hasFailure, tt.wantFailure) + } + + // If we expect failure, check the failure output content + if tt.wantFailure && hasFailure { + for _, want := range tt.wantFailureContains { + if !strings.Contains(failureOutput, want) { + t.Errorf("failure output missing expected string %q\nGot:\n%s", want, failureOutput) + } + } + + // Print the actual failure output for review + t.Logf("Failure output:\n%s", failureOutput) + } + }) + } +}