diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50e92a2..03c4a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,6 +172,61 @@ jobs: tests/artifacts/baselines/ retention-days: 7 + e2e-test: + name: E2E Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + + - name: Install asciinema (Linux) + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y asciinema + # Install Xvfb for potential TUI tests + sudo apt-get install -y xvfb + + - name: Install asciinema (macOS) + if: matrix.os == 'macos-latest' + run: brew install asciinema + + - name: Configure PowerShell for E2E tests (Windows) + if: matrix.os == 'windows-latest' + shell: powershell + run: Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope Process + + - name: Run E2E tests + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + # Use Xvfb to run tests in a virtual display environment + xvfb-run go test -v -timeout=300s ./tests/e2e/... + else + go test -v -timeout=300s ./tests/e2e/... + fi + + - name: Upload E2E artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-results-${{ matrix.os }} + path: | + tests/artifacts/e2e/ + retention-days: 30 + test-matrix: name: Comprehensive Testing Matrix runs-on: ubuntu-latest @@ -325,12 +380,12 @@ jobs: notify: name: Notify runs-on: ubuntu-latest - needs: [test-matrix, coverage-report, performance-benchmark, security-scan] + needs: [test, e2e-test, test-matrix, coverage-report, performance-benchmark, security-scan] if: always() && (github.event_name == 'push' || github.event_name == 'pull_request') steps: - name: Notify on test failure - if: needs.test.result == 'failure' || needs.test-matrix.result == 'failure' + if: needs.test.result == 'failure' || needs.e2e-test.result == 'failure' || needs.test-matrix.result == 'failure' run: | echo "::warning ::Some tests failed. Please check the logs above." @@ -340,6 +395,6 @@ jobs: echo "::warning ::Coverage generation failed or threshold not met." - name: Success notification - if: needs.test.result == 'success' && needs.test-matrix.result == 'success' + if: needs.test.result == 'success' && needs.e2e-test.result == 'success' && needs.test-matrix.result == 'success' run: | echo "::notice ::All tests passed successfully!" diff --git a/docs/testing-quickstart.md b/docs/testing-quickstart.md index a505ec6..5188806 100644 --- a/docs/testing-quickstart.md +++ b/docs/testing-quickstart.md @@ -45,6 +45,9 @@ make test-unit # Integration tests only make test-integration +# E2E tests only +go test -v ./tests/e2e/... + # Benchmarks only make test-benchmark @@ -91,7 +94,10 @@ make quick-bench tests/ ├── unit/ # Fast, isolated unit tests ├── integration/ # Component interaction tests -├── e2e/ # End-to-end tests (future) +├── e2e/ # End-to-end tests simulating user workflows +├── recording/ # Terminal recording utilities +├── cross_platform/ # Cross-platform compatibility tests +├── reporting/ # Test reporting generation └── artifacts/ # Test reports and coverage ``` @@ -103,6 +109,9 @@ cd tests/unit && go test -v -cover # Run integration tests cd tests/integration && go test -v +# Run E2E tests +cd tests/e2e && go test -v + # Run tests in parallel cd tests/unit && go test -parallel=4 ``` @@ -131,6 +140,23 @@ open tests/artifacts/coverage/coverage.html cat tests/artifacts/coverage/coverage_summary.txt ``` +## E2E Testing + +### Running E2E Tests +To run the end-to-end tests, you can execute the following command: +```bash +# Run all E2E tests +go test -v ./tests/e2e/... +``` + +### E2E Artifacts +After running the E2E tests, you can find the terminal recordings in the `tests/artifacts/e2e/` directory. Each test run will have its own subdirectory containing the `.cast` file for the recording. + +An HTML report summarizing all E2E test runs is also generated. You can find it at: +``` +tests/artifacts/reports/e2e_report.html +``` + ## Performance Testing ### Run Benchmarks @@ -231,10 +257,10 @@ make monitor-performance ### GitHub Actions The testing framework is integrated with GitHub Actions and automatically: -- Runs tests on Linux, macOS, and Windows +- Runs unit, integration, and E2E tests on Linux, macOS, and Windows - Generates coverage reports (Unix only) - Runs benchmarks (Linux only) -- Uploads test artifacts +- Uploads test and E2E artifacts, including terminal recordings - Checks coverage thresholds ### Quality Gates diff --git a/docs/testing-standards.md b/docs/testing-standards.md index a764096..791c066 100644 --- a/docs/testing-standards.md +++ b/docs/testing-standards.md @@ -27,7 +27,7 @@ Unit Tests (70%) - **Unit Tests (70%)**: Fast, isolated tests of individual functions and methods - **Integration Tests (25%)**: Tests of component interactions and integration points -- **E2E Tests (5%)**: Full workflow tests (framework established for future use) +- **E2E Tests (5%)**: Full workflow tests that simulate real user scenarios from start to finish across all supported platforms. ## Test Organization Structure @@ -43,17 +43,22 @@ tests/ │ ├── component_interaction_test.go # Component interaction scenarios │ ├── integration_fixtures.go # Test fixtures and mock objects │ └── isolation_test.go # Test isolation and cleanup procedures -├── e2e/ # End-to-end tests (framework for 0.2.3) -│ └── workflow_test.go # Complete workflow testing +├── e2e/ # End-to-end tests simulating user workflows +│ ├── workflow_test.go # Core user journey tests +│ ├── platform_test.go # Platform-specific E2E tests +│ └── e2e_utils.go # E2E test utilities and orchestration +├── recording/ # Terminal recording utilities and tests +│ ├── capture_utils.go # Core recording functionality +│ └── capture_test.go # Tests for recording utilities +├── cross_platform/ # Cross-platform compatibility tests +│ └── platform_compat_test.go # Tests for platform-specific behavior +├── reporting/ # Test reporting generation and utilities +│ ├── reporting_utils.go # Report generation logic +│ └── aggregator_test.go # Tests for report data aggregation ├── artifacts/ # Generated test evidence and reports │ ├── coverage/ # Coverage reports and historical data -│ │ ├── coverage.html # HTML coverage report -│ │ ├── coverage_summary.txt # Text coverage summary -│ │ └── combined_coverage.out # Combined coverage profile -│ ├── reports/ # Test execution reports -│ │ ├── benchmark_results.txt # Benchmark results -│ │ ├── performance_metrics.txt # Performance metrics -│ │ └── unix_test_report.txt # Unix-specific test report +│ ├── e2e/ # E2E test recordings and reports +│ ├── reports/ # Other test execution reports │ └── platform_results/ # Platform-specific test results └── scripts/ # Test execution utilities ├── unix_test_tools.sh # Unix-specific testing tools @@ -100,6 +105,38 @@ func TestCalculatorAddition(t *testing.T) { } ``` +### End-to-End (E2E) Tests + +**Purpose**: Validate complete user workflows from start to finish across all supported platforms. + +**Characteristics**: +- Slower execution time, as they involve building and running the actual application binary. +- Test the application as a whole, including its interaction with the operating system. +- Provide terminal recordings as artifacts for visual validation and debugging. + +**Best Practices**: +```go +// Good E2E test +func TestE2EWorkflow(t *testing.T) { + t.Run("TestSimpleAddition", func(t *testing.T) { + run := NewE2ETestRun(t) + + expression := "2 + 2" + expected := "4" + + // Record the command + run.RecordCommand(expression) + + // Run the command for assertion + output := run.RunCommand(expression) + + if !strings.Contains(output, expected) { + t.Errorf("Expected output to contain '%s', but got '%s'", expected, output) + } + }) +} +``` + ## Test Types and Categories ### Unit Tests @@ -313,7 +350,7 @@ The Unix environment validation script checks: |----------|---------------|-----------------|------------|-------| | Linux (Ubuntu) | ✅ Full | ✅ HTML + Text | ✅ Detailed | Primary platform | | macOS | ✅ Full | ✅ HTML + Text | ✅ Basic | Development platform | -| Windows | ✅ Core | ❌ Limited | ❌ Limited | CI validation only | +| Windows | ✅ Full | ❌ Limited | ❌ Limited | Full CI validation including E2E | ### CI Pipeline @@ -492,8 +529,8 @@ make coverage-html ### Planned Features -1. **Visual Testing**: Screenshot capture and comparison (Story 0.2.2) -2. **E2E Testing**: Complete workflow testing (Story 0.2.3) +1. **Visual Testing**: Screenshot capture and comparison (✅ **Completed** in Story 0.2.2) +2. **E2E Testing**: Complete workflow testing (✅ **Completed** in Story 0.2.3) 3. **Performance Regression**: Historical performance tracking 4. **Test Analytics**: Test execution analytics and insights 5. **Advanced CI/CD**: Enhanced CI/CD features and integrations diff --git a/tests/cross_platform/platform_compat_test.go b/tests/cross_platform/platform_compat_test.go new file mode 100644 index 0000000..a677bbe --- /dev/null +++ b/tests/cross_platform/platform_compat_test.go @@ -0,0 +1,79 @@ +package cross_platform + +import ( + "os" + "runtime" + "strings" + "testing" + + "github.com/dmisiuk/acousticalc/tests/recording" +) + +func TestPlatformIdentification(t *testing.T) { + t.Run("TestIdentifyOperatingSystem", func(t *testing.T) { + switch runtime.GOOS { + case "linux": + t.Log("Running on Linux") + case "darwin": + t.Log("Running on macOS") + case "windows": + t.Log("Running on Windows") + default: + t.Logf("Running on an unrecognized operating system: %s", runtime.GOOS) + } + }) + + t.Run("TestPathSeparator", func(t *testing.T) { + expectedSeparator := "/" + if runtime.GOOS == "windows" { + expectedSeparator = "\\" + } + + // In Go, os.PathSeparator is a rune (int32). We need to convert it to a string for comparison. + actualSeparator := string(os.PathSeparator) + + if actualSeparator != expectedSeparator { + t.Errorf("Expected path separator to be '%s' on %s, but got '%s'", expectedSeparator, runtime.GOOS, actualSeparator) + } else { + t.Logf("Correct path separator ('%s') found for %s", actualSeparator, runtime.GOOS) + } + }) + + t.Run("TestRecordingMechanism", func(t *testing.T) { + tempDir := t.TempDir() + recorder := recording.NewRecorder(tempDir, "TestRecording") + + var expectedExtension string + if runtime.GOOS == "windows" { + expectedExtension = ".txt" + } else { + expectedExtension = ".cast" + } + + // Use 'go version' as a simple, cross-platform command that produces output. + err := recorder.RecordCommand("go", "version") + if err != nil { + t.Fatalf("RecordCommand failed: %v", err) + } + + files, err := os.ReadDir(tempDir) + if err != nil { + t.Fatalf("Failed to read temp dir: %v", err) + } + + if len(files) != 1 { + var createdFiles []string + for _, f := range files { + createdFiles = append(createdFiles, f.Name()) + } + t.Fatalf("Expected 1 file in temp dir, but got %d. Files: %v", len(files), createdFiles) + } + + createdFile := files[0].Name() + if !strings.HasSuffix(createdFile, expectedExtension) { + t.Errorf("Expected file with extension '%s', but got '%s'", expectedExtension, createdFile) + } else { + t.Logf("Correct recording artifact found: %s", createdFile) + } + }) +} diff --git a/tests/e2e/e2e_utils.go b/tests/e2e/e2e_utils.go new file mode 100644 index 0000000..e5c899b --- /dev/null +++ b/tests/e2e/e2e_utils.go @@ -0,0 +1,114 @@ +package e2e + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/dmisiuk/acousticalc/tests/recording" +) + +const ( + binaryName = "acousticalc" +) + +var ( + binaryPath string +) + +// E2ETestRun manages the context for a single E2E test run. +type E2ETestRun struct { + t *testing.T + testName string + outputDir string + recorder *recording.Recorder +} + +// NewE2ETestRun creates a new E2E test run. +func NewE2ETestRun(t *testing.T) *E2ETestRun { + testName := strings.ReplaceAll(t.Name(), "/", "_") + platform := runtime.GOOS + outputDir := filepath.Join("../../tests/artifacts/e2e", platform, testName) + + if err := os.MkdirAll(outputDir, 0755); err != nil { + t.Fatalf("Failed to create artifact directory '%s': %v", outputDir, err) + } + + recDir := filepath.Join(outputDir, "recordings") + + return &E2ETestRun{ + t: t, + testName: testName, + outputDir: outputDir, + recorder: recording.NewRecorder(recDir, testName), + } +} + +// RecordCommand runs the acousticalc binary within a recording session for its side effect. +func (r *E2ETestRun) RecordCommand(args ...string) { + r.t.Helper() + + if binaryPath == "" { + r.t.Fatal("E2E test setup failed: binaryPath is not set. Was TestMain run correctly?") + } + + err := r.recorder.RecordCommand(binaryPath, args...) + if err != nil { + r.t.Fatalf("Failed to record command: %v", err) + } + r.t.Logf("Command recorded successfully.") +} + +// RunCommand runs the acousticalc binary and returns its output for assertions. +func (r *E2ETestRun) RunCommand(args ...string) string { + r.t.Helper() + + if binaryPath == "" { + r.t.Fatal("E2E test setup failed: binaryPath is not set. Was TestMain run correctly?") + } + + cmd := exec.Command(binaryPath, args...) + output, err := cmd.CombinedOutput() + + if err != nil { + r.t.Fatalf("Failed to run command for assertion: %v, output: %s", err, output) + } + + return strings.TrimSpace(string(output)) +} + +// setupE2ETests builds the binary for the E2E tests and returns a teardown function. +func setupE2ETests() (func(), error) { + tempDir, err := os.MkdirTemp("", "acousticalc-e2e-") + if err != nil { + return nil, fmt.Errorf("failed to create temp dir for binary: %w", err) + } + + var binName string + if runtime.GOOS == "windows" { + binName = binaryName + ".exe" + } else { + binName = binaryName + } + + path := filepath.Join(tempDir, binName) + + cmd := exec.Command("go", "build", "-o", path, "../../cmd/acousticalc") + buildOutput, err := cmd.CombinedOutput() + if err != nil { + os.RemoveAll(tempDir) + return nil, fmt.Errorf("failed to build binary: %v\nOutput: %s", err, buildOutput) + } + + binaryPath = path + + teardown := func() { + os.RemoveAll(tempDir) + } + + return teardown, nil +} diff --git a/tests/e2e/workflow_test.go b/tests/e2e/workflow_test.go new file mode 100644 index 0000000..c6716cd --- /dev/null +++ b/tests/e2e/workflow_test.go @@ -0,0 +1,58 @@ +package e2e + +import ( + "os" + "strings" + "testing" +) + +// TestMain sets up the E2E test environment. +func TestMain(m *testing.M) { + teardown, err := setupE2ETests() + if err != nil { + os.Stderr.WriteString("Failed to setup E2E tests: " + err.Error() + "\n") + os.Exit(1) + } + + exitCode := m.Run() + + teardown() + + os.Exit(exitCode) +} + +func TestE2EWorkflow(t *testing.T) { + t.Run("TestSimpleAddition", func(t *testing.T) { + run := NewE2ETestRun(t) + + expression := "2 + 2" + expected := "4" + + // Record the command + run.RecordCommand(expression) + + // Run the command for assertion + output := run.RunCommand(expression) + + if !strings.Contains(output, expected) { + t.Errorf("Expected output to contain '%s', but got '%s'", expected, output) + } + }) + + t.Run("TestComplexExpression", func(t *testing.T) { + run := NewE2ETestRun(t) + + expression := "(5 + 3) * 2 - 4" + expected := "12" + + // Record the command + run.RecordCommand(expression) + + // Run the command for assertion + output := run.RunCommand(expression) + + if !strings.Contains(output, expected) { + t.Errorf("Expected output to contain '%s', but got '%s'", expected, output) + } + }) +} diff --git a/tests/recording/capture_utils.go b/tests/recording/capture_utils.go new file mode 100644 index 0000000..ec36744 --- /dev/null +++ b/tests/recording/capture_utils.go @@ -0,0 +1,80 @@ +package recording + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +// Recorder handles the terminal recording process. +type Recorder struct { + outputDir string + testName string +} + +// NewRecorder creates a new recorder instance. +func NewRecorder(outputDir, testName string) *Recorder { + return &Recorder{ + outputDir: outputDir, + testName: testName, + } +} + +// RecordCommand executes a command within a recording session. +// It uses asciinema on Linux/macOS and PowerShell Start-Transcript on Windows. +func (r *Recorder) RecordCommand(command string, args ...string) error { + if err := os.MkdirAll(r.outputDir, 0755); err != nil { + return fmt.Errorf("failed to create recording output directory: %w", err) + } + + if runtime.GOOS == "windows" { + return r.recordWithPowerShell(command, args...) + } + return r.recordWithAsciinema(command, args...) +} + +func (r *Recorder) recordWithAsciinema(command string, args ...string) error { + filename := fmt.Sprintf("%s_%s.cast", r.testName, time.Now().Format("20060102_150405")) + filePath := filepath.Join(r.outputDir, filename) + + fullCommand := fmt.Sprintf("%s %s", command, strings.Join(args, " ")) + + // Use --quiet to suppress asciinema's own output. + cmd := exec.Command("asciinema", "rec", "--overwrite", "--quiet", "-c", fullCommand, filePath) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("asciinema recording failed: %w", err) + } + return nil +} + +func (r *Recorder) recordWithPowerShell(command string, args ...string) error { + filename := fmt.Sprintf("%s_%s.txt", r.testName, time.Now().Format("20060102_150405")) + filePath := filepath.Join(r.outputDir, filename) + + // Escape arguments for PowerShell + var escapedArgs []string + for _, arg := range args { + escapedArgs = append(escapedArgs, fmt.Sprintf("'%s'", strings.ReplaceAll(arg, "'", "''"))) + } + + // Construct the PowerShell command string + // Start-Transcript logs the session. + // & executes the command. + // Stop-Transcript stops logging. + // We use -join to handle expressions with spaces correctly. + fullCommand := fmt.Sprintf("& '%s' %s", command, strings.Join(escapedArgs, " ")) + psCommand := fmt.Sprintf("Start-Transcript -Path '%s' -NoClobber; try { %s } finally { Stop-Transcript }", filePath, fullCommand) + + cmd := exec.Command("powershell.exe", "-NoProfile", "-Command", psCommand) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("powershell recording failed: %w\nOutput: %s", err, string(output)) + } + return nil +} diff --git a/tests/reporting/aggregator_test.go b/tests/reporting/aggregator_test.go new file mode 100644 index 0000000..2a60317 --- /dev/null +++ b/tests/reporting/aggregator_test.go @@ -0,0 +1,122 @@ +package reporting + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestE2EReporting(t *testing.T) { + // Create a temporary directory for our fake artifacts + artifactsDir, err := os.MkdirTemp("", "test-artifacts-") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(artifactsDir) + + // --- Setup fake artifacts for Linux and Windows --- + e2eDir := filepath.Join(artifactsDir, "e2e") + + // Linux artifact + linuxTestDir := filepath.Join(e2eDir, "linux", "TestE2EWorkflow_TestSimpleAddition", "recordings") + if err := os.MkdirAll(linuxTestDir, 0755); err != nil { + t.Fatalf("Failed to create linux test dir: %v", err) + } + linuxRecPath := filepath.Join(linuxTestDir, "rec1.cast") + if _, err := os.Create(linuxRecPath); err != nil { + t.Fatalf("Failed to create linux rec: %v", err) + } + + // Windows artifact + windowsTestDir := filepath.Join(e2eDir, "windows", "TestE2EWorkflow_TestComplexExpression", "recordings") + if err := os.MkdirAll(windowsTestDir, 0755); err != nil { + t.Fatalf("Failed to create windows test dir: %v", err) + } + windowsRecPath := filepath.Join(windowsTestDir, "rec2.txt") + if _, err := os.Create(windowsRecPath); err != nil { + t.Fatalf("Failed to create windows rec: %v", err) + } + + t.Run("TestDiscoverCrossPlatformArtifacts", func(t *testing.T) { + // Run the discovery function + reportData, err := DiscoverE2EArtifacts(artifactsDir) + if err != nil { + t.Fatalf("DiscoverE2EArtifacts failed: %v", err) + } + + // Assert the results + if len(reportData.TestRuns) != 2 { + t.Fatalf("Expected 2 test runs, but got %d", len(reportData.TestRuns)) + } + + // Check for the Linux run + foundLinuxRun := false + for _, run := range reportData.TestRuns { + if run.Platform == "linux" { + foundLinuxRun = true + if run.Name != "TestE2EWorkflow_TestSimpleAddition" { + t.Errorf("Incorrect test name for linux run: got %s", run.Name) + } + if !strings.HasSuffix(run.Recording, ".cast") { + t.Errorf("Incorrect recording file for linux run: got %s", run.Recording) + } + } + } + if !foundLinuxRun { + t.Error("Did not find the linux test run") + } + + // Check for the Windows run + foundWindowsRun := false + for _, run := range reportData.TestRuns { + if run.Platform == "windows" { + foundWindowsRun = true + if run.Name != "TestE2EWorkflow_TestComplexExpression" { + t.Errorf("Incorrect test name for windows run: got %s", run.Name) + } + if !strings.HasSuffix(run.Recording, ".txt") { + t.Errorf("Incorrect recording file for windows run: got %s", run.Recording) + } + } + } + if !foundWindowsRun { + t.Error("Did not find the windows test run") + } + }) + + t.Run("TestGenerateE2EReportWithCrossPlatformData", func(t *testing.T) { + // First, get the data + reportData, err := DiscoverE2EArtifacts(artifactsDir) + if err != nil { + t.Fatalf("DiscoverE2EArtifacts failed: %v", err) + } + + // Generate the report + reportOutputDir := filepath.Join(artifactsDir, "reports") + reportPath, err := GenerateE2EReport(reportData, reportOutputDir) + if err != nil { + t.Fatalf("GenerateE2EReport failed: %v", err) + } + + // Verify the report content + content, err := os.ReadFile(reportPath) + if err != nil { + t.Fatalf("Failed to read report file: %v", err) + } + htmlContent := string(content) + + if !strings.Contains(htmlContent, "linux") { + t.Error("Report does not contain the linux platform") + } + if !strings.Contains(htmlContent, "windows") { + t.Error("Report does not contain the windows platform") + } + if !strings.Contains(htmlContent, ".cast") { + t.Error("Report does not contain a link to the .cast file") + } + if !strings.Contains(htmlContent, ".txt") { + t.Error("Report does not contain a link to the .txt file") + } + }) +} diff --git a/tests/reporting/reporting_utils.go b/tests/reporting/reporting_utils.go new file mode 100644 index 0000000..f752e99 --- /dev/null +++ b/tests/reporting/reporting_utils.go @@ -0,0 +1,135 @@ +package reporting + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// E2EReportData holds the aggregated data for the E2E test report. +type E2EReportData struct { + TestRuns []TestRun +} + +// TestRun represents a single E2E test execution. +type TestRun struct { + Name string + Platform string + Recording string // Relative path to the recording file +} + +// DiscoverE2EArtifacts scans the artifacts directory to find all E2E test runs. +func DiscoverE2EArtifacts(artifactsDir string) (*E2EReportData, error) { + reportData := &E2EReportData{ + TestRuns: make([]TestRun, 0), + } + + e2eArtifactsDir := filepath.Join(artifactsDir, "e2e") + if _, err := os.Stat(e2eArtifactsDir); os.IsNotExist(err) { + // It's not an error if the directory doesn't exist, just means no E2E tests were run + return reportData, nil + } + + // Walk through the E2E artifacts directory + err := filepath.Walk(e2eArtifactsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Check if the file is a recording (.cast for Unix, .txt for Windows) + if !info.IsDir() && (strings.HasSuffix(info.Name(), ".cast") || strings.HasSuffix(info.Name(), ".txt")) { + // Make path relative to the e2e artifacts dir to easily extract platform and test name + relativePath, err := filepath.Rel(e2eArtifactsDir, path) + if err != nil { + return err // Or log a warning + } + + parts := strings.Split(relativePath, string(os.PathSeparator)) + + // Expected structure: //recordings/.cast + if len(parts) >= 4 { + platform := parts[0] + testName := parts[1] + + // Make the recording path relative to the root artifacts directory for the HTML report + reportRelativePath, err := filepath.Rel(artifactsDir, path) + if err != nil { + reportRelativePath = path // Fallback to full path + } + + run := TestRun{ + Name: testName, + Platform: platform, + Recording: reportRelativePath, + } + reportData.TestRuns = append(reportData.TestRuns, run) + } + } + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk E2E artifacts directory: %w", err) + } + + return reportData, nil +} + +// GenerateE2EReport creates an HTML report from the aggregated E2E test data. +func GenerateE2EReport(reportData *E2EReportData, outputDir string) (string, error) { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create report output directory: %w", err) + } + + reportPath := filepath.Join(outputDir, "e2e_report.html") + htmlContent := generateHTMLContent(reportData) + + if err := os.WriteFile(reportPath, []byte(htmlContent), 0644); err != nil { + return "", fmt.Errorf("failed to write E2E report: %w", err) + } + + return reportPath, nil +} + +func generateHTMLContent(data *E2EReportData) string { + html := ` + + + E2E Test Report + + + +
+

E2E Test Report

+

Total Test Runs: ` + fmt.Sprintf("%d", len(data.TestRuns)) + `

+
+ + + + + + ` + + for _, run := range data.TestRuns { + html += fmt.Sprintf(` + + + + + `, run.Name, run.Platform, run.Recording) + } + + html += ` +
Test NamePlatformRecording
%s%sView Recording
+ +` + return html +} diff --git a/tests/scripts/Makefile b/tests/scripts/Makefile index 10a07d4..8f8fde2 100644 --- a/tests/scripts/Makefile +++ b/tests/scripts/Makefile @@ -21,7 +21,13 @@ UNIX_SCRIPT := $(TESTS_DIR)/scripts/unix_test_tools.sh # Unix-specific detection UNAME_S := $(shell uname -s) -NPROC := $(shell nproc) +ifeq ($(UNAME_S),Linux) + NPROC := $(shell nproc) +else ifeq ($(UNAME_S),Darwin) + NPROC := $(shell sysctl -n hw.ncpu) +else + NPROC := 1 +endif # Environment variables export PARALLEL_JOBS ?= $(NPROC) @@ -44,7 +50,7 @@ test-integration: ## Run integration tests with Unix optimizations test-benchmark: ## Run performance benchmarks @echo "⚡ Running performance benchmarks..." @cd $(TESTS_DIR)/unit && $(GO) test -bench=. -benchmem -timeout=120s \ - -parallel=$(PARALLEL_JOBS) ./... > $(ARTIFACTS_DIR/reports/benchmark_results.txt 2>&1 + -parallel=$(PARALLEL_JOBS) ./... > $(ARTIFACTS_DIR)/reports/benchmark_results.txt 2>&1 test-coverage: coverage-html coverage-summary ## Generate coverage reports @@ -70,7 +76,7 @@ coverage-html: ## Generate HTML coverage report @mkdir -p $(ARTIFACTS_DIR)/coverage @if [ -f $(ARTIFACTS_DIR)/coverage/unit_coverage.out ]; then \ $(GO) tool cover -html=$(ARTIFACTS_DIR)/coverage/unit_coverage.out \ - -o $(ARTIFACTS_DIR/coverage/coverage.html; \ + -o $(ARTIFACTS_DIR)/coverage/coverage.html; \ echo "✅ HTML coverage report: $(ARTIFACTS_DIR)/coverage/coverage.html"; \ else \ echo "⚠️ No coverage file found, run tests first"; \ @@ -80,7 +86,7 @@ coverage-summary: ## Generate coverage summary @echo "📈 Generating coverage summary..." @if [ -f $(ARTIFACTS_DIR)/coverage/unit_coverage.out ]; then \ $(GO) tool cover -func=$(ARTIFACTS_DIR)/coverage/unit_coverage.out \ - > $(ARTIFACTS_DIR/coverage/coverage_summary.txt; \ + > $(ARTIFACTS_DIR)/coverage/coverage_summary.txt; \ echo "✅ Coverage summary: $(ARTIFACTS_DIR)/coverage/coverage_summary.txt"; \ grep "total:" $(ARTIFACTS_DIR)/coverage/coverage_summary.txt; \ else \ @@ -91,8 +97,8 @@ coverage-summary: ## Generate coverage summary benchmark-detailed: ## Run detailed benchmarks with analysis @echo "🔬 Running detailed benchmarks..." @cd $(TESTS_DIR)/unit && $(GO) test -bench=. -benchmem -benchtime=5s \ - -count=3 -timeout=300s ./... > $(ARTIFACTS_DIR/reports/detailed_benchmarks.txt 2>&1 - @echo "✅ Detailed benchmarks: $(ARTIFACTS_DIR/reports/detailed_benchmarks.txt" + -count=3 -timeout=300s ./... > $(ARTIFACTS_DIR)/reports/detailed_benchmarks.txt 2>&1 + @echo "✅ Detailed benchmarks: $(ARTIFACTS_DIR)/reports/detailed_benchmarks.txt" # Setup targets setup-unix: ## Setup Unix-specific testing environment @@ -110,12 +116,12 @@ setup-completion: ## Setup shell completion unix-report: ## Generate Unix-specific test report @echo "📋 Generating Unix test report..." @$(UNIX_SCRIPT) validate && echo "✅ Unix environment validated" - @echo "System Information:" > $(ARTIFACTS_DIR/reports/unix_system_info.txt - @uname -a >> $(ARTIFACTS_DIR/reports/unix_system_info.txt - @echo "Go Version:" >> $(ARTIFACTS_DIR/reports/unix_system_info.txt - @$(GO) version >> $(ARTIFACTS_DIR/reports/unix_system_info.txt - @echo "Parallel Jobs: $(PARALLEL_JOBS)" >> $(ARTIFACTS_DIR/reports/unix_system_info.txt - @echo "✅ Unix system report: $(ARTIFACTS_DIR/reports/unix_system_info.txt" + @echo "System Information:" > $(ARTIFACTS_DIR)/reports/unix_system_info.txt + @uname -a >> $(ARTIFACTS_DIR)/reports/unix_system_info.txt + @echo "Go Version:" >> $(ARTIFACTS_DIR)/reports/unix_system_info.txt + @$(GO) version >> $(ARTIFACTS_DIR)/reports/unix_system_info.txt + @echo "Parallel Jobs: $(PARALLEL_JOBS)" >> $(ARTIFACTS_DIR)/reports/unix_system_info.txt + @echo "✅ Unix system report: $(ARTIFACTS_DIR)/reports/unix_system_info.txt" # Performance monitoring monitor-performance: ## Monitor system performance during tests diff --git a/tests/scripts/unix_test_tools.sh b/tests/scripts/unix_test_tools.sh index facbcbd..827f32d 100755 --- a/tests/scripts/unix_test_tools.sh +++ b/tests/scripts/unix_test_tools.sh @@ -293,18 +293,30 @@ run_all_tests() { monitor_performance() { log_info "Monitoring test performance..." - # Monitor CPU usage during tests - local cpu_usage=$(top -l 1 -n 0 | grep "CPU usage" | awk '{print $3}' | sed 's/%//') - local memory_usage=$(vm_stat | grep "Pages free" | awk '{print $3}' | sed 's/\.$//') + local cpu_usage="N/A" + local memory_usage="N/A" + local memory_usage_label="Memory Free" + + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS-specific commands + cpu_usage=$(top -l 1 -n 0 | grep "CPU usage" | awk '{print $3}' | sed 's/%//') + memory_usage=$(vm_stat | grep "Pages free" | awk '{print $3}' | sed 's/\.$//') + memory_usage_label="Memory Free Pages" + elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + # Linux-specific commands + cpu_usage=$(top -bn1 | grep '%Cpu(s)' | awk '{print $2+$4}') + memory_usage=$(free -m | awk '/^Mem:/{print $4}') + memory_usage_label="Memory Free (MB)" + fi log_info "CPU Usage: ${cpu_usage}%" - log_info "Memory Free Pages: ${memory_usage}" + log_info "${memory_usage_label}: ${memory_usage}" # Save performance metrics { echo "Performance Metrics - $(date)" echo "CPU Usage: ${cpu_usage}%" - echo "Memory Free Pages: ${memory_usage}" + echo "${memory_usage_label}: ${memory_usage}" echo "Parallel Jobs: $PARALLEL_JOBS" echo "Coverage Enabled: $COVERAGE_ENABLED" } >> "$ARTIFACTS_DIR/reports/performance_metrics.txt"