diff --git a/Cargo.lock b/Cargo.lock index e7c415ec..9fc25b2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -810,6 +810,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -1356,6 +1365,7 @@ dependencies = [ "candle-nn", "chrono", "flate2", + "html-escape", "log", "ordered-float", "parking_lot", @@ -2043,6 +2053,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 299e32c2..bba87a1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ rand_distr = "0.5.1" parking_lot = "0.12.4" rand_chacha = "0.9.0" rayon = "1.10.0" +html-escape = "0.2.13" [features] default = ["plotting"] diff --git a/docs/unified_reporting.md b/docs/unified_reporting.md new file mode 100644 index 00000000..f349b13f --- /dev/null +++ b/docs/unified_reporting.md @@ -0,0 +1,263 @@ +# Unified Report System + +This document describes the unified reporting system for the QQN Optimizer benchmark suite. The system provides a consistent interface for generating various types of performance reports across multiple output formats. + +## Overview + +The unified reporting system is built around the `Report` trait, which provides a standardized interface for all report types. This allows for: + +- **Consistent API**: All reports implement the same interface +- **Multiple formats**: HTML, LaTeX, Markdown, and CSV output +- **Unified testing**: Common test patterns for all report implementations +- **Easy extension**: Simple to add new report types +- **Batch processing**: Generate multiple reports at once + +## Core Components + +### Report Trait + +The `Report` trait defines the interface that all reports must implement: + +```rust +pub trait Report { + fn name(&self) -> &'static str; + fn description(&self) -> &'static str; + fn generate_content(&self, data: &[(&ProblemSpec, BenchmarkResults)], config: &ReportConfig) -> Result; + fn export_to_file(&self, data: &[(&ProblemSpec, BenchmarkResults)], config: &ReportConfig, output_path: &Path) -> Result<()>; + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> Result<()>; + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata; + fn supported_formats(&self) -> Vec; +} +``` + +### Report Configuration + +Reports are configured using the `ReportConfig` struct: + +```rust +pub struct ReportConfig { + pub format: ReportFormat, // Output format (HTML, LaTeX, etc.) + pub include_detailed_stats: bool, // Include detailed statistics + pub include_plots: bool, // Include visualizations + pub style_options: HashMap, // Custom styling +} +``` + +### Report Collection + +The `ReportCollection` struct allows batch processing of multiple reports: + +```rust +let reports = ReportCollection::new() + .add_report(SummaryStatisticsReport::new()) + .add_report(PerformanceTableReport::new()); + +let metadata = reports.generate_all(&data, &config, &output_dir)?; +``` + +## Available Report Types + +### Summary Statistics Report + +Provides aggregate performance metrics grouped by problem family and optimizer. + +### Family vs Family Report +Shows a comparison matrix of how different optimizer families perform across different problem families. +- **Name**: `family_vs_family` +- **Formats**: HTML, LaTeX, Markdown, CSV +- **Use case**: Cross-family performance comparison + + +### Performance Table Report + +Shows detailed performance metrics for each optimizer-problem combination. + + +## Usage Examples + +### Basic Report Generation + +```rust +use qqn_optimizer::experiment_runner::{Report, ReportConfig, ReportFormat}; +use qqn_optimizer::experiment_runner::reports::unified_summary_statistics::SummaryStatisticsReport; + +let report = SummaryStatisticsReport::new(); +let config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() +}; + +let content = report.generate_content(&benchmark_data, &config)?; +``` + +### Batch Report Generation + +```rust +use qqn_optimizer::experiment_runner::{ReportCollection, ReportConfig, ReportFormat}; + +let reports = ReportCollection::new() + .add_report(SummaryStatisticsReport::new()) + .add_report(PerformanceTableReport::new()) + .add_report(FamilyVsFamilyReport::new()); + +let config = ReportConfig { + format: ReportFormat::Html, + include_detailed_stats: true, + include_plots: false, + ..Default::default() +}; + +let metadata_list = reports.generate_all(&data, &config, &output_dir)?; +``` + +### Multiple Format Generation + +```rust +let report = SummaryStatisticsReport::new(); +let formats = [ReportFormat::Html, ReportFormat::Markdown, ReportFormat::Csv]; + +for format in formats { + let config = ReportConfig { format, ..Default::default() }; + let content = report.generate_content(&data, &config)?; + // Save content to appropriate file... +} +``` + +## Testing Infrastructure + +The unified reporting system includes comprehensive testing infrastructure through the `UnifiedReportTestSuite`: + +### Test Categories + +1. **Basic Functionality**: Tests report name, description, supported formats +2. **Content Generation**: Validates content generation for all formats +3. **Data Validation**: Tests input data validation and error handling +4. **Metadata Generation**: Validates report metadata +5. **File Export**: Tests file export functionality +6. **Format Consistency**: Ensures different formats produce different but valid content + +### Running Tests + +```rust +use qqn_optimizer::experiment_runner::UnifiedReportTestSuite; + +// Test a specific report +let report = SummaryStatisticsReport::new(); +UnifiedReportTestSuite::test_report(&report)?; + +// Test individual aspects +UnifiedReportTestSuite::test_basic_functionality(&report)?; +UnifiedReportTestSuite::test_content_generation(&report)?; +// ... etc +``` + +### Creating Test Data + +The test suite provides utilities for creating test data: + +```rust +let test_data = UnifiedReportTestSuite::create_test_data(); +let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); +``` + +## Adding New Report Types + +To add a new report type: + +1. **Create the Report Struct**: +```rust +pub struct MyCustomReport; + +impl MyCustomReport { + pub fn new() -> Self { + Self + } +} +``` + +2. **Implement the Report Trait**: +```rust +impl Report for MyCustomReport { + fn name(&self) -> &'static str { + "my_custom_report" + } + + fn description(&self) -> &'static str { + "Description of what this report provides" + } + + fn generate_content(&self, data: &[(&ProblemSpec, BenchmarkResults)], config: &ReportConfig) -> Result { + // Implementation for different formats + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + // ... etc + } + } +} +``` + +3. **Add Tests**: +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::UnifiedReportTestSuite; + + #[test] + fn test_my_custom_report() { + let report = MyCustomReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } +} +``` + +4. **Add to Collection** (optional): +```rust +let reports = ReportCollection::new() + .add_report(MyCustomReport::new()) + .add_report(SummaryStatisticsReport::new()); +``` + +## Output Formats + +### HTML +- Complete standalone HTML documents +- Embedded CSS for styling +- Tables and basic formatting +- Suitable for web display + +### LaTeX +- Complete LaTeX documents with packages +- Professional table formatting +- Scientific notation support +- Ready for academic publication + +### Markdown +- GitHub-flavored markdown +- Table support +- Suitable for documentation +- Easy to convert to other formats + +### CSV +- Comma-separated values +- Easy data import/export +- Suitable for further analysis +- Compatible with spreadsheet software + +## Performance Considerations + +- Reports validate input data before processing +- Large datasets are processed efficiently +- File export uses streaming where appropriate +- Memory usage is optimized for batch processing + +## Future Extensions + +The unified reporting system is designed to be extensible: + +- **New Formats**: Easy to add JSON, XML, or other formats +- **Enhanced Styling**: More sophisticated styling options +- **Interactive Reports**: Support for interactive HTML reports +- **Template System**: Customizable report templates +- **Caching**: Results caching for large datasets \ No newline at end of file diff --git a/src/benchmarks/evaluation.rs b/src/benchmarks/evaluation.rs index 4c0b2013..f666709c 100644 --- a/src/benchmarks/evaluation.rs +++ b/src/benchmarks/evaluation.rs @@ -221,6 +221,17 @@ pub struct PerformanceMetrics { pub convergence_rate: f64, } +impl PerformanceMetrics { + pub(crate) fn default() -> PerformanceMetrics { + PerformanceMetrics { + iterations_per_second: 0.0, + function_evaluations_per_second: 0.0, + gradient_evaluations_per_second: 0.0, + convergence_rate: 0.0, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum ConvergenceReason { GradientTolerance, diff --git a/src/experiment_runner/mod.rs b/src/experiment_runner/mod.rs index 99541a90..6dd7aa54 100644 --- a/src/experiment_runner/mod.rs +++ b/src/experiment_runner/mod.rs @@ -5,10 +5,25 @@ pub mod optimizer_sets; pub mod plotting_manager; pub mod problem_sets; pub mod report_generator; -mod reports; +pub mod reports; pub mod statistical_analysis; +pub mod unified_report; +pub mod unified_report_tests; +pub use experiment_runner::*; +pub use report_generator::*; +pub use reports::convergence_analysis::ConvergenceAnalysisReport; +pub use reports::efficiency_matrix::EfficiencyMatrixReport; +pub use reports::family_vs_family_report::FamilyVsFamilyReport; +pub use reports::heatmap::SuccessRateHeatmapReport; +pub use reports::unified_performance_table::PerformanceTableReport; +pub use reports::unified_summary_statistics::SummaryStatisticsReport; +pub use statistical_analysis::*; +pub use unified_report::*; +pub mod unified_report_example; pub use experiment_runner::ExperimentRunner; pub use plotting_manager::PlottingManager; pub use report_generator::ReportGenerator; pub use statistical_analysis::StatisticalAnalysis; +pub use unified_report::{Report, ReportCollection, ReportConfig, ReportFormat, ReportMetadata}; +pub use unified_report_tests::UnifiedReportTestSuite; diff --git a/src/experiment_runner/report_generator.rs b/src/experiment_runner/report_generator.rs index bb5daab5..01fa8eb4 100644 --- a/src/experiment_runner/report_generator.rs +++ b/src/experiment_runner/report_generator.rs @@ -9,15 +9,19 @@ use crate::experiment_runner::reports::comparison_matrix::{ generate_comparison_matrix_latex_table, generate_comparison_matrix_table_content, generate_family_comparison_matrix_table_content, }; +use crate::experiment_runner::reports::convergence_analysis::ConvergenceAnalysisReport; use crate::experiment_runner::reports::convergence_analysis::{ generate_convergence_analysis, generate_convergence_speed_latex_table, generate_convergence_speed_table_content, }; use crate::experiment_runner::reports::efficiency_matrix::generate_efficiency_matrix_latex_table; +use crate::experiment_runner::reports::efficiency_matrix::EfficiencyMatrixReport; use crate::experiment_runner::reports::family_vs_family::{ generate_family_vs_family_comparison_table, generate_family_vs_family_latex_table, generate_family_vs_family_table_content, }; +use crate::experiment_runner::reports::family_vs_family_report::FamilyVsFamilyReport; +use crate::experiment_runner::reports::heatmap::SuccessRateHeatmapReport; use crate::experiment_runner::reports::heatmap::{ generate_success_rate_heatmap_latex_table, generate_success_rate_heatmap_table_content, }; @@ -28,6 +32,11 @@ use crate::experiment_runner::reports::performance_table::{ use crate::experiment_runner::reports::summary_statistics::{ generate_summary_statistics_latex_table, generate_summary_statistics_table_content, }; +use crate::experiment_runner::reports::unified_performance_table::PerformanceTableReport; +use crate::experiment_runner::reports::unified_summary_statistics::SummaryStatisticsReport; +use crate::experiment_runner::unified_report::{ + Report, ReportCollection, ReportConfig, ReportFormat, +}; use crate::OptimizationProblem; use anyhow::Context; use log::warn; @@ -92,6 +101,190 @@ impl ReportGenerator { statistical_analysis: StatisticalAnalysis::new(), } } + /// Generate reports using the unified reporting system + pub async fn generate_unified_reports( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + formats: &[ReportFormat], + ) -> anyhow::Result<()> { + let reports_dir = Path::new(&self.output_dir).join("unified_reports"); + fs::create_dir_all(&reports_dir)?; + println!( + "Generating unified reports in directory: {}", + reports_dir.display() + ); + // Create report collection with all available reports + let collection = ReportCollection::new() + .add_report(ConvergenceAnalysisReport::new()) + .add_report(EfficiencyMatrixReport::new()) + .add_report(SuccessRateHeatmapReport::new()) + .add_report(PerformanceTableReport::new()) + .add_report(SummaryStatisticsReport::new()) + .add_report(FamilyVsFamilyReport::new()); + // Generate reports in each requested format + for format in formats { + let format_dir = reports_dir.join(format!("{:?}", format).to_lowercase()); + fs::create_dir_all(&format_dir)?; + let config = ReportConfig { + format: format.clone(), + include_detailed_stats: true, + include_plots: true, + style_options: std::collections::HashMap::new(), + }; + let metadata = collection.generate_all(all_results, &config, &format_dir)?; + // Log generation results + println!( + "Generated {} reports in {:?} format:", + metadata.len(), + format + ); + for meta in &metadata { + println!( + " - {}: {} problems, {} optimizers, {} data points", + meta.report_type, meta.problem_count, meta.optimizer_count, meta.data_points + ); + } + } + Ok(()) + } + /// Generate a comprehensive report index that links to all unified reports + pub async fn generate_report_index( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + formats: &[ReportFormat], + ) -> anyhow::Result<()> { + let index_path = Path::new(&self.output_dir).join("report_index.html"); + let mut html_content = String::from( + r#" + + + + + QQN Benchmark Report Index + + + +

QQN Benchmark Report Index

+

Generated on: "#, + ); + html_content.push_str( + &chrono::Utc::now() + .format("%Y-%m-%d %H:%M:%S UTC") + .to_string(), + ); + html_content.push_str("

"); + // Add summary statistics + let total_problems = all_results.len(); + let total_runs: usize = all_results.iter().map(|(_, r)| r.results.len()).sum(); + let unique_optimizers: std::collections::HashSet<_> = all_results + .iter() + .flat_map(|(_, r)| &r.results) + .map(|result| &result.optimizer_name) + .collect(); + html_content.push_str(&format!( + r#"
+

Benchmark Summary

+
    +
  • Total Problems: {}
    • +
  • Total Optimizers: {}
    • +
  • Total Runs: {}
    • +
  • Available Formats: {}
    • +
+
+"#, + total_problems, + unique_optimizers.len(), + total_runs, + formats + .iter() + .map(|f| format!("{:?}", f)) + .collect::>() + .join(", ") + )); + // Add unified reports section + html_content.push_str( + r#"

Unified Reports

+

Modern, standardized reports with consistent formatting across all output types.

+
+"#, + ); + let report_descriptions = vec![ + ("convergence_analysis", "Convergence Analysis", "Analyzes convergence speed and patterns across optimizers, showing mean iterations to reach improvement milestones"), + ("efficiency_matrix", "Efficiency Matrix", "Algorithm efficiency matrix showing mean function evaluations for successful runs across problem families"), + ("success_rate_heatmap", "Success Rate Heatmap", "Color-coded heatmap showing success rates across optimizer-problem combinations"), + ("performance_table", "Performance Table", "Detailed performance table showing metrics for each optimizer-problem combination"), + ("summary_statistics", "Summary Statistics", "Summary statistics showing average performance metrics grouped by problem family and optimizer"), + ("family_vs_family", "Family vs Family", "Comparison matrix showing how different optimizer families perform across different problem families"), + ]; + for (report_name, display_name, description) in report_descriptions { + html_content.push_str(&format!( + r#"
+

{}

+

{}

+
+"#, + display_name, description + )); + for format in formats { + let format_str = format!("{:?}", format).to_lowercase(); + let extension = match format { + ReportFormat::Html => "html", + ReportFormat::Latex => "tex", + ReportFormat::Markdown => "md", + ReportFormat::Csv => "csv", + }; + html_content.push_str(&format!( + r#" {:?} +"#, + format_str, report_name, extension, format + )); + } + html_content.push_str( + r#"
+
+"#, + ); + } + html_content.push_str("
"); + // Add legacy reports section + html_content.push_str( + r#"
+

Legacy Reports

+

Traditional report formats for backward compatibility.

+ +
+"#, + ); + html_content.push_str( + r#"
+

Generated by QQN Optimizer Benchmark Suite

+
+ + +"#, + ); + fs::write(&index_path, html_content).with_context(|| { + format!("Failed to write report index to: {}", index_path.display()) + })?; + println!("Generated report index: {}", index_path.display()); + Ok(()) + } pub async fn generate_main_report( &self, @@ -100,6 +293,18 @@ impl ReportGenerator { ) -> anyhow::Result<()> { fs::create_dir_all(&self.output_dir) .with_context(|| format!("Failed to create output directory: {}", self.output_dir))?; + // Generate unified reports in multiple formats + let unified_formats = vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ]; + self.generate_unified_reports(all_results, &unified_formats) + .await?; + self.generate_report_index(all_results, &unified_formats) + .await?; + // Create hierarchical directory structure let reports_dir = Path::new(&self.output_dir).join("reports"); let data_dir = Path::new(&self.output_dir).join("data"); @@ -158,9 +363,68 @@ impl ReportGenerator { generate_latex_tables(&latex_dir.to_string_lossy(), all_results, self).await?; // Generate comprehensive LaTeX document generate_comprehensive_latex_document(&self.config, all_results, &latex_dir, self)?; + println!("Report generation complete!"); + println!(" - Unified reports: {}/unified_reports/", self.output_dir); + println!(" - Report index: {}/report_index.html", self.output_dir); + println!( + " - Legacy reports: {}/benchmark_report.md", + self.output_dir + ); + println!(" - LaTeX tables: {}/latex/", self.output_dir); + println!(" - Raw data: {}/data/", self.output_dir); Ok(()) } + /// Generate only unified reports (for testing or when legacy reports are not needed) + pub async fn generate_unified_only( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + formats: Option>, + ) -> anyhow::Result<()> { + fs::create_dir_all(&self.output_dir) + .with_context(|| format!("Failed to create output directory: {}", self.output_dir))?; + let formats = formats.unwrap_or_else(|| { + vec![ + ReportFormat::Html, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + }); + self.generate_unified_reports(all_results, &formats).await?; + self.generate_report_index(all_results, &formats).await?; + println!("Unified report generation complete!"); + println!(" - Reports: {}/unified_reports/", self.output_dir); + println!(" - Index: {}/report_index.html", self.output_dir); + Ok(()) + } + /// Get available unified report types + pub fn get_available_unified_reports() -> Vec<(&'static str, &'static str)> { + vec![ + ("convergence_analysis", "Convergence Analysis"), + ("efficiency_matrix", "Efficiency Matrix"), + ("success_rate_heatmap", "Success Rate Heatmap"), + ("performance_table", "Performance Table"), + ("summary_statistics", "Summary Statistics"), + ("family_vs_family", "Family vs Family Comparison"), + ] + } + /// Generate a specific unified report + pub async fn generate_specific_unified_report( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + report: R, + format: ReportFormat, + ) -> anyhow::Result { + let config = ReportConfig { + format, + include_detailed_stats: true, + include_plots: true, + style_options: std::collections::HashMap::new(), + }; + report.validate_data(all_results)?; + let content = report.generate_content(all_results, &config)?; + Ok(content) + } } /// Generate efficiency matrix table content (without document wrapper) diff --git a/src/experiment_runner/reports/convergence_analysis.rs b/src/experiment_runner/reports/convergence_analysis.rs index 96c96258..ab4b48a8 100644 --- a/src/experiment_runner/reports/convergence_analysis.rs +++ b/src/experiment_runner/reports/convergence_analysis.rs @@ -1,12 +1,383 @@ use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec, SingleResult}; -use crate::experiment_runner::report_generator; +use crate::experiment_runner::{ + report_generator, Report, ReportConfig, ReportFormat, ReportMetadata, +}; use anyhow::Context; +use std::collections::HashMap; use std::fs; use std::path::Path; +/// Convergence Analysis Report +pub struct ConvergenceAnalysisReport; +impl ConvergenceAnalysisReport { + pub fn new() -> Self { + Self + } + fn generate_html( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + let mut content = String::from( + r#" + + + Convergence Analysis Report + + + +

Convergence Analysis Report

+"#, + ); + if config.include_detailed_stats { + content.push_str(&self.generate_convergence_speed_html_table(data)?); + } + content.push_str(""); + Ok(content) + } + fn generate_latex( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + let mut content = String::from( + r#"\documentclass{article} +\usepackage{booktabs} +\usepackage{array} +\usepackage[table]{xcolor} +\usepackage{siunitx} +\usepackage{adjustbox} +\usepackage[margin=1in]{geometry} +\usepackage{float} +\begin{document} +\title{Convergence Analysis Report} +\maketitle +"#, + ); + if config.include_detailed_stats { + content.push_str(&self.generate_convergence_speed_latex_table(data)?); + } + content.push_str(r#"\end{document}"#); + Ok(content) + } + fn generate_markdown( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + let mut content = String::from("# Convergence Analysis Report\n\n"); + if config.include_detailed_stats { + content.push_str(&self.generate_convergence_speed_markdown_table(data)?); + } + Ok(content) + } + fn generate_csv( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let mut content = String::from( + "Optimizer,Mean_Iterations_50%,Mean_Iterations_90%,Final_Convergence_Iteration\n", + ); + let optimizer_averages = self.calculate_convergence_averages(data)?; + for (optimizer, avg_50, avg_90, avg_final) in optimizer_averages { + content.push_str(&format!( + "{},{:.1},{:.1},{:.1}\n", + optimizer, avg_50, avg_90, avg_final + )); + } + Ok(content) + } + fn calculate_convergence_averages( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + ) -> anyhow::Result> { + let mut optimizer_speed_data = HashMap::new(); + for (_, results) in data { + for result in &results.results { + if result.convergence_achieved && !result.trace.iterations.is_empty() { + let speed_data = optimizer_speed_data + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + let initial_value = result + .trace + .iterations + .first() + .map(|iter| iter.function_value) + .unwrap_or(result.final_value); + let final_value = result.final_value; + let improvement_50 = initial_value - 0.5 * (initial_value - final_value); + let improvement_90 = initial_value - 0.9 * (initial_value - final_value); + let mut iter_to_50 = None; + let mut iter_to_90 = None; + for iter_data in &result.trace.iterations { + if iter_to_50.is_none() && iter_data.function_value <= improvement_50 { + iter_to_50 = Some(iter_data.iteration); + } + if iter_to_90.is_none() && iter_data.function_value <= improvement_90 { + iter_to_90 = Some(iter_data.iteration); + } + } + speed_data.push(( + iter_to_50.unwrap_or(result.iterations), + iter_to_90.unwrap_or(result.iterations), + result.iterations, + )); + } + } + } + let mut optimizer_averages = Vec::new(); + for (optimizer, speed_data) in optimizer_speed_data { + if !speed_data.is_empty() { + let avg_50 = speed_data + .iter() + .map(|(iter_50, _, _)| *iter_50 as f64) + .sum::() + / speed_data.len() as f64; + let avg_90 = speed_data + .iter() + .map(|(_, iter_90, _)| *iter_90 as f64) + .sum::() + / speed_data.len() as f64; + let avg_final = speed_data + .iter() + .map(|(_, _, final_iter)| *final_iter as f64) + .sum::() + / speed_data.len() as f64; + optimizer_averages.push((optimizer, avg_50, avg_90, avg_final)); + } + } + // Sort by fastest overall convergence (weighted average of milestones) + optimizer_averages.sort_by(|a, b| { + let score_a = 0.3 * a.1 + 0.4 * a.2 + 0.3 * a.3; + let score_b = 0.3 * b.1 + 0.4 * b.2 + 0.3 * b.3; + score_a + .partial_cmp(&score_b) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(optimizer_averages) + } + fn generate_convergence_speed_html_table( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + ) -> anyhow::Result { + let optimizer_averages = self.calculate_convergence_averages(data)?; + if optimizer_averages.is_empty() { + return Ok(String::new()); + } + let mut content = String::from( + r#" +

Convergence Speed Analysis

+ + + + + + + + + + +"#, + ); + for (i, (optimizer, avg_50, avg_90, avg_final)) in optimizer_averages.iter().enumerate() { + let class = if i == 0 { + "best" + } else if optimizer.contains("QQN") { + "qqn" + } else { + "" + }; + content.push_str(&format!( + r#" + + + + + +"#, + class, optimizer, avg_50, avg_90, avg_final + )); + } + content.push_str(r#" +
OptimizerMean Iterations to 50% ImprovementMean Iterations to 90% ImprovementFinal Convergence Iteration
{}{:.1}{:.1}{:.1}
+

Purpose: Compares convergence rates for different optimizers. Sorted by fastest overall convergence (weighted average). Best performer is highlighted in bold, QQN variants in green.

+"#); + Ok(content) + } + fn generate_convergence_speed_markdown_table( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + ) -> anyhow::Result { + let optimizer_averages = self.calculate_convergence_averages(data)?; + if optimizer_averages.is_empty() { + return Ok(String::new()); + } + let mut content = String::from( + r#"## Convergence Speed Analysis +| Optimizer | Mean Iterations to 50% Improvement | Mean Iterations to 90% Improvement | Final Convergence Iteration | +|-----------|-------------------------------------|-------------------------------------|------------------------------| +"#, + ); + for (optimizer, avg_50, avg_90, avg_final) in optimizer_averages { + content.push_str(&format!( + "| {} | {:.1} | {:.1} | {:.1} |\n", + optimizer, avg_50, avg_90, avg_final + )); + } + content.push_str("\n**Purpose:** Compares convergence rates for different optimizers. Sorted by fastest overall convergence (weighted average).\n\n"); + Ok(content) + } + + fn generate_convergence_speed_latex_table( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + ) -> anyhow::Result { + // Convert HTML table to LaTeX format + let html_content = self.generate_convergence_speed_html_table(data)?; + // Simple conversion - you might want to make this more sophisticated + let latex_content = html_content + .replace("", "\\begin{tabular}{|l|c|c|c|}\n\\hline") + .replace("
", "\\end{tabular}") + .replace("", "") + .replace("", " \\\\\n\\hline") + .replace("", "") + .replace("", " & ") + .replace("", "") + .replace("", " & "); + Ok(latex_content) + } +} +impl Report for ConvergenceAnalysisReport { + fn name(&self) -> &'static str { + "convergence_analysis" + } + fn description(&self) -> &'static str { + "Analyzes convergence speed and patterns across optimizers, showing mean iterations to reach improvement milestones" + } + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> anyhow::Result<()> { + let content = self.generate_content(data, config)?; + fs::write(output_path, content).with_context(|| { + format!( + "Failed to write convergence analysis report to: {}", + output_path.display() + ) + })?; + Ok(()) + } + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> anyhow::Result<()> { + if data.is_empty() { + return Err(anyhow::anyhow!("No benchmark data provided")); + } + let has_convergence_data = data.iter().any(|(_, results)| { + results + .results + .iter() + .any(|r| r.convergence_achieved && !r.trace.iterations.is_empty()) + }); + if !has_convergence_data { + return Err(anyhow::anyhow!( + "No convergence data available for analysis" + )); + } + Ok(()) + } + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let total_problems = data.len(); + let total_runs: usize = data.iter().map(|(_, results)| results.results.len()).sum(); + let convergent_runs: usize = data + .iter() + .map(|(_, results)| { + results + .results + .iter() + .filter(|r| r.convergence_achieved) + .count() + }) + .sum(); + let mut metadata = HashMap::new(); + metadata.insert("total_problems".to_string(), total_problems.to_string()); + metadata.insert("total_runs".to_string(), total_runs.to_string()); + metadata.insert("convergent_runs".to_string(), convergent_runs.to_string()); + metadata.insert( + "convergence_rate".to_string(), + format!( + "{:.1}%", + (convergent_runs as f64 / total_runs as f64) * 100.0 + ), + ); + let optimizer_count = data + .iter() + .flat_map(|(_, results)| &results.results) + .map(|r| &r.optimizer_name) + .collect::>() + .len(); + let data_points = data.iter().map(|(_, results)| results.results.len()).sum(); + + ReportMetadata { + report_type: self.name().to_string(), + generated_at: chrono::Utc::now(), + problem_count: total_problems, + optimizer_count, + data_points, + } + } + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} +impl Default for ConvergenceAnalysisReport { + fn default() -> Self { + Self::new() + } +} +// Legacy function for backward compatibility - now uses the unified report /// Generate convergence speed table content (without document wrapper) +#[deprecated(note = "Use ConvergenceAnalysisReport::new().generate_content() instead")] pub fn generate_convergence_speed_table_content( all_results: &[(&ProblemSpec, BenchmarkResults)], +) -> anyhow::Result { + let report = ConvergenceAnalysisReport::new(); + let config = ReportConfig { + format: ReportFormat::Latex, + include_detailed_stats: true, + ..Default::default() + }; + report.generate_content(all_results, &config) +} +// Keep the original implementation for backward compatibility +fn generate_convergence_speed_table_content_legacy( + all_results: &[(&ProblemSpec, BenchmarkResults)], ) -> anyhow::Result { // Similar logic as generate_convergence_speed_latex_table but return just the table content let mut optimizer_speed_data = std::collections::HashMap::new(); @@ -113,6 +484,7 @@ pub fn generate_convergence_speed_table_content( ); Ok(content) } +#[deprecated(note = "Use ConvergenceAnalysisReport for unified reporting")] pub fn generate_convergence_analysis(runs: &[&SingleResult]) -> anyhow::Result { let successful_runs: Vec<_> = runs.iter().filter(|r| r.convergence_achieved).collect(); @@ -186,6 +558,7 @@ pub fn generate_convergence_analysis(runs: &[&SingleResult]) -> anyhow::Result anyhow::Result<()> { - // Collect all optimizer families and problem families - let mut all_optimizer_families = std::collections::HashSet::new(); - let mut all_problem_families = std::collections::HashSet::new(); - for (problem, results) in all_results { - let problem_family = report_generator::get_family(&problem.get_name()); - all_problem_families.insert(problem_family); - for result in &results.results { - let optimizer_family = get_optimizer_family(&result.optimizer_name); - all_optimizer_families.insert(optimizer_family); +impl EfficiencyMatrixReport { + pub fn new() -> Self { + Self + } + + fn collect_families( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + ) -> (Vec, Vec) { + let mut all_optimizer_families = std::collections::HashSet::new(); + let mut all_problem_families = std::collections::HashSet::new(); + + for (problem, results) in all_results { + let problem_family = report_generator::get_family(&problem.get_name()); + all_problem_families.insert(problem_family); + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + all_optimizer_families.insert(optimizer_family); + } } + + let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); + let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); + optimizer_families.sort(); + problem_families.sort(); + + (optimizer_families, problem_families) } - let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); - let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); - optimizer_families.sort(); - problem_families.sort(); - let mut all_optimizer_families = std::collections::HashSet::new(); - let mut all_problem_families = std::collections::HashSet::new(); - for (problem, results) in all_results { - let problem_family = report_generator::get_family(&problem.get_name()); - all_problem_families.insert(problem_family); + fn calculate_efficiency_data( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + ) -> HashMap<(String, String), (f64, f64, usize)> { + let mut efficiency_data = HashMap::new(); + let (optimizer_families, problem_families) = self.collect_families(all_results); + + for optimizer_family in &optimizer_families { + for problem_family in &problem_families { + let mut successful_evaluations = Vec::new(); + + for (problem, results) in all_results { + if report_generator::get_family(&problem.get_name()) == *problem_family { + for result in &results.results { + let result_optimizer_family = + get_optimizer_family(&result.optimizer_name); + if result_optimizer_family == *optimizer_family + && result.convergence_achieved + { + successful_evaluations.push(result.function_evaluations as f64); + } + } + } + } - for result in &results.results { - let optimizer_family = get_optimizer_family(&result.optimizer_name); - all_optimizer_families.insert(optimizer_family); + if !successful_evaluations.is_empty() { + let mean = successful_evaluations.iter().sum::() + / successful_evaluations.len() as f64; + let variance = successful_evaluations + .iter() + .map(|x| (x - mean).powi(2)) + .sum::() + / successful_evaluations.len() as f64; + let std_dev = variance.sqrt(); + efficiency_data.insert( + (optimizer_family.clone(), problem_family.clone()), + (mean, std_dev, successful_evaluations.len()), + ); + } + } } + + efficiency_data } - let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); - let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); - optimizer_families.sort(); - problem_families.sort(); + fn generate_html( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizer_families, problem_families) = self.collect_families(all_results); + let efficiency_data = self.calculate_efficiency_data(all_results); + + if optimizer_families.is_empty() || problem_families.is_empty() { + return Ok("

No data available for efficiency matrix.

".to_string()); + } + + let mut html_content = String::from( + r#" + + + Algorithm Efficiency Matrix + + + +

Algorithm Efficiency Matrix

+

Mean Function Evaluations for Successful Runs

+ + + + "#, + ); + + for problem_family in &problem_families { + html_content.push_str(&format!("", problem_family)); + } + html_content.push_str(""); + + for optimizer_family in &optimizer_families { + html_content.push_str(&format!( + "", + optimizer_family + )); + + for problem_family in &problem_families { + let cell_content = if let Some((mean, std_dev, _count)) = + efficiency_data.get(&(optimizer_family.clone(), problem_family.clone())) + { + format!("{:.0} ± {:.0}", mean, std_dev) + } else { + "N/A".to_string() + }; + html_content.push_str(&format!("", cell_content)); + } + html_content.push_str(""); + } + + html_content.push_str( + r#" +
Optimizer Family{}
{}{}
+

Purpose: Shows mean function evaluations ± standard deviation for successful runs only across problem families. Lower values indicate higher efficiency.

+ +"# + ); - if optimizer_families.is_empty() || problem_families.is_empty() { - return Ok(()); + Ok(html_content) } - // Calculate column specification dynamically - let col_spec = format!("l{}", "c".repeat(problem_families.len())); - let mut latex_content = String::from( - r#"\documentclass{article} + fn generate_latex( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizer_families, problem_families) = self.collect_families(all_results); + let efficiency_data = self.calculate_efficiency_data(all_results); + + if optimizer_families.is_empty() || problem_families.is_empty() { + return Ok("No data available for efficiency matrix.".to_string()); + } + + let col_spec = format!("l{}", "c".repeat(problem_families.len())); + + let mut latex_content = String::from( + r#"\documentclass{article} \usepackage{booktabs} \usepackage{array} \usepackage{xcolor} @@ -59,9 +182,10 @@ pub fn generate_efficiency_matrix_latex_table( \usepackage[margin=1in]{geometry} \begin{document} "#, - ); - latex_content.push_str(&format!( - r#"\begin{{table}}[htbp] + ); + + latex_content.push_str(&format!( + r#"\begin{{table}}[htbp] \centering \caption{{Algorithm Efficiency Matrix: Mean Function Evaluations for Successful Runs}} \label{{tab:efficiency_matrix}} @@ -71,69 +195,279 @@ pub fn generate_efficiency_matrix_latex_table( \textbf{{Optimizer Family}} {}\\ \midrule "#, - problem_families - .iter() - .map(|fam| format!("& \\textbf{{{}}}", report_generator::escape_latex(fam))) - .collect::>() - .join(" ") - )); - // Calculate efficiency data for each optimizer family across problem families - for optimizer_family in &optimizer_families { - latex_content.push_str(&format!( - "\\textbf{{{}}} ", - report_generator::escape_latex(optimizer_family) + problem_families + .iter() + .map(|fam| format!("& \\textbf{{{}}}", report_generator::escape_latex(fam))) + .collect::>() + .join(" ") )); - for problem_family in &problem_families { - // Get all successful runs for this optimizer family on this problem family - let mut successful_evaluations = Vec::new(); - for (problem, results) in all_results { - if report_generator::get_family(&problem.get_name()) == *problem_family { - for result in &results.results { - let result_optimizer_family = get_optimizer_family(&result.optimizer_name); - if result_optimizer_family == *optimizer_family - && result.convergence_achieved - { - successful_evaluations.push(result.function_evaluations as f64); - } - } - } + + for optimizer_family in &optimizer_families { + latex_content.push_str(&format!( + "\\textbf{{{}}} ", + report_generator::escape_latex(optimizer_family) + )); + + for problem_family in &problem_families { + let cell_content = if let Some((mean, std_dev, _count)) = + efficiency_data.get(&(optimizer_family.clone(), problem_family.clone())) + { + format!("{:.0} $\\pm$ {:.0}", mean, std_dev) + } else { + "N/A".to_string() + }; + latex_content.push_str(&format!("& {} ", cell_content)); } - let cell_content = if successful_evaluations.is_empty() { - "N/A".to_string() - } else { - let mean = successful_evaluations.iter().sum::() - / successful_evaluations.len() as f64; - let variance = successful_evaluations - .iter() - .map(|x| (x - mean).powi(2)) - .sum::() - / successful_evaluations.len() as f64; - let std_dev = variance.sqrt(); - format!("{mean:.0} $\\pm$ {std_dev:.0}") - }; - latex_content.push_str(&format!("& {cell_content} ")); - } - latex_content.push_str("\\\\\n"); - } - latex_content.push_str( - r#"\bottomrule + latex_content.push_str("\\\\\n"); + } + + latex_content.push_str( + r#"\bottomrule \end{tabular} } \end{table} \textbf{Purpose:} Shows mean function evaluations $\pm$ standard deviation for successful runs only across problem families. Lower values indicate higher efficiency. \end{document} "#, - ); + ); + + Ok(latex_content) + } + + fn generate_markdown( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizer_families, problem_families) = self.collect_families(all_results); + let efficiency_data = self.calculate_efficiency_data(all_results); + + if optimizer_families.is_empty() || problem_families.is_empty() { + return Ok("No data available for efficiency matrix.".to_string()); + } + + let mut markdown_content = String::from("# Algorithm Efficiency Matrix\n\n"); + markdown_content.push_str("Mean Function Evaluations for Successful Runs\n\n"); + + // Table header + markdown_content.push_str("| Optimizer Family |"); + for problem_family in &problem_families { + markdown_content.push_str(&format!(" {} |", problem_family)); + } + markdown_content.push_str("\n|"); + + // Table separator + markdown_content.push_str("---|"); + for _ in &problem_families { + markdown_content.push_str("---|"); + } + markdown_content.push_str("\n"); + + // Table rows + for optimizer_family in &optimizer_families { + markdown_content.push_str(&format!("| **{}** |", optimizer_family)); + + for problem_family in &problem_families { + let cell_content = if let Some((mean, std_dev, _count)) = + efficiency_data.get(&(optimizer_family.clone(), problem_family.clone())) + { + format!("{:.0} ± {:.0}", mean, std_dev) + } else { + "N/A".to_string() + }; + markdown_content.push_str(&format!(" {} |", cell_content)); + } + markdown_content.push_str("\n"); + } + + markdown_content.push_str("\n**Purpose:** Shows mean function evaluations ± standard deviation for successful runs only across problem families. Lower values indicate higher efficiency.\n"); + + Ok(markdown_content) + } + + fn generate_csv( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizer_families, problem_families) = self.collect_families(all_results); + let efficiency_data = self.calculate_efficiency_data(all_results); + + if optimizer_families.is_empty() || problem_families.is_empty() { + return Ok("No data available for efficiency matrix.".to_string()); + } + + let mut csv_content = String::from("Optimizer Family"); + for problem_family in &problem_families { + csv_content.push_str(&format!(",{}", problem_family)); + } + csv_content.push_str("\n"); + + for optimizer_family in &optimizer_families { + csv_content.push_str(optimizer_family); + + for problem_family in &problem_families { + let cell_content = if let Some((mean, std_dev, _count)) = + efficiency_data.get(&(optimizer_family.clone(), problem_family.clone())) + { + format!("{:.0} ± {:.0}", mean, std_dev) + } else { + "N/A".to_string() + }; + csv_content.push_str(&format!(",{}", cell_content)); + } + csv_content.push_str("\n"); + } + + Ok(csv_content) + } +} + +impl Report for EfficiencyMatrixReport { + fn name(&self) -> &'static str { + "efficiency_matrix" + } + + fn description(&self) -> &'static str { + "Algorithm Efficiency Matrix showing mean function evaluations for successful runs across problem families" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> anyhow::Result<()> { + let content = self.generate_content(data, config)?; + fs::write(output_path, content).with_context(|| { + format!( + "Failed to write efficiency matrix report to: {}", + output_path.display() + ) + })?; + Ok(()) + } + + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> anyhow::Result<()> { + if data.is_empty() { + return Err(anyhow::anyhow!("No benchmark data provided")); + } + + // Check that we have at least some results with convergence data + let has_convergence_data = data + .iter() + .any(|(_, results)| results.results.iter().any(|r| r.convergence_achieved)); + + if !has_convergence_data { + return Err(anyhow::anyhow!( + "No convergence data found in benchmark results" + )); + } + + Ok(()) + } + + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let (optimizer_families, problem_families) = self.collect_families(data); + let efficiency_data = self.calculate_efficiency_data(data); + + let total_successful_runs: usize = + efficiency_data.values().map(|(_, _, count)| count).sum(); + + let mut metadata = HashMap::new(); + metadata.insert( + "optimizer_families".to_string(), + optimizer_families.len().to_string(), + ); + metadata.insert( + "problem_families".to_string(), + problem_families.len().to_string(), + ); + metadata.insert( + "total_successful_runs".to_string(), + total_successful_runs.to_string(), + ); + metadata.insert( + "matrix_cells".to_string(), + (optimizer_families.len() * problem_families.len()).to_string(), + ); + + ReportMetadata { + report_type: self.name().to_string(), + generated_at: chrono::Utc::now(), + problem_count: data.len(), + optimizer_count: data + .iter() + .flat_map(|(_, results)| &results.results) + .map(|r| &r.optimizer_name) + .collect::>() + .len(), + data_points: data.iter().map(|(_, results)| results.results.len()).sum(), + } + } + + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} + +/// Legacy function for backward compatibility +/// Generate efficiency matrix LaTeX table +pub fn generate_efficiency_matrix_latex_table( + all_results: &[(&ProblemSpec, BenchmarkResults)], + latex_dir: &Path, +) -> anyhow::Result<()> { + let report = EfficiencyMatrixReport::new(); + let config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + let latex_path = latex_dir.join("efficiency_matrix.tex"); - fs::write(&latex_path, latex_content).with_context(|| { - format!( - "Failed to write efficiency matrix LaTeX table to: {}", - latex_path.display() - ) - })?; + report.export_to_file(all_results, &config, &latex_path)?; + println!( "Generated efficiency matrix LaTeX table: {}", latex_path.display() ); + Ok(()) } +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::UnifiedReportTestSuite; + #[test] + fn test_efficiency_matrix_report() { + let report = EfficiencyMatrixReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + #[test] + fn test_efficiency_matrix_basic_functionality() { + let report = EfficiencyMatrixReport::new(); + UnifiedReportTestSuite::test_basic_functionality(&report).unwrap(); + } + #[test] + fn test_efficiency_matrix_content_generation() { + let report = EfficiencyMatrixReport::new(); + UnifiedReportTestSuite::test_content_generation(&report).unwrap(); + } +} diff --git a/src/experiment_runner/reports/family_vs_family.rs b/src/experiment_runner/reports/family_vs_family.rs index c4e0c61c..a113b4c5 100644 --- a/src/experiment_runner/reports/family_vs_family.rs +++ b/src/experiment_runner/reports/family_vs_family.rs @@ -1,7 +1,7 @@ use crate::benchmarks::evaluation::{is_no_threshold_mode, BenchmarkResults, ProblemSpec}; use crate::experiment_runner::experiment_runner::get_optimizer_family; use crate::experiment_runner::report_generator; -use crate::experiment_runner::report_generator::FamilyPerformanceData; +use crate::experiment_runner::report_generator::{escape_latex_safe, FamilyPerformanceData}; use anyhow::Context; use std::collections::HashMap; use std::fs; @@ -882,7 +882,7 @@ Generated on: {} ## Test Data: The test uses mock benchmark results for the following problem families: - **rosenbrock**: rosenbrock_2d, rosenbrock_10d -- **sphere**: sphere_2d, sphere_10d +- **sphere**: sphere_2d, sphere_10d - **rastrigin**: rastrigin_2d And the following optimizer families: - **lbfgs**: lbfgs_default, lbfgs_aggressive diff --git a/src/experiment_runner/reports/family_vs_family_report.rs b/src/experiment_runner/reports/family_vs_family_report.rs new file mode 100644 index 00000000..c5844c04 --- /dev/null +++ b/src/experiment_runner/reports/family_vs_family_report.rs @@ -0,0 +1,588 @@ +//! Family vs Family comparison report implementation for the unified reporting system. +//! +//! This report provides a comprehensive comparison matrix showing how different +//! optimizer families perform across different problem families. + +use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; +use crate::experiment_runner::experiment_runner::get_optimizer_family; +use crate::experiment_runner::report_generator; +use crate::experiment_runner::reports::family_vs_family::{ + calculate_family_performance_data, generate_family_vs_family_comparison_table, + generate_family_vs_family_table_content, +}; +use crate::experiment_runner::unified_report::{ + Report, ReportConfig, ReportFormat, ReportMetadata, +}; +use anyhow::Result; +use std::collections::HashSet; +use std::path::Path; + +/// Family vs Family comparison report +pub struct FamilyVsFamilyReport; + +impl FamilyVsFamilyReport { + /// Create a new family vs family report + pub fn new() -> Self { + Self + } + + /// Generate HTML content for the report + fn generate_html( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut content = String::from( + r#" + + + + + Family vs Family Performance Matrix + + + +

Family vs Family Performance Matrix

+"#, + ); + + // Generate HTML table content directly + self.generate_html_table_content(data, &mut content)?; + + content.push_str("\n"); + + Ok(content) + } + /// Generate HTML table content for the family vs family comparison + fn generate_html_table_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + content: &mut String, + ) -> Result<()> { + // Collect all optimizer families and problem families + let mut all_optimizer_families = HashSet::new(); + let mut all_problem_families = HashSet::new(); + for (problem, results) in data { + let problem_family = report_generator::get_family(&problem.get_name()); + all_problem_families.insert(problem_family); + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + all_optimizer_families.insert(optimizer_family); + } + } + let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); + let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); + optimizer_families.sort(); + problem_families.sort(); + if optimizer_families.is_empty() || problem_families.is_empty() { + content.push_str("

No data available for family comparison.

\n"); + return Ok(()); + } + content.push_str(r#"

This table shows how different optimizer families perform across different problem families. Each cell contains:

+
    +
  • Avg Rank: Average ranking of all variants in the optimizer family across problems in the problem family (lower is better)
    • +
  • Best Rank: Average of the best rank achieved by any variant in the optimizer family for each problem
    • +
  • Best Var: The specific optimizer variant that achieved the best average rank
    • +
  • Worst Var: The specific optimizer variant that achieved the worst average rank
    • +
+"#); + // Create the table header + content.push_str(r#" + + +"#); + for optimizer_family in &optimizer_families { + content.push_str(&format!( + r#" +"#, + optimizer_family + )); + } + content.push_str("\n"); + // For each problem family, calculate statistics + for problem_family in &problem_families { + content.push_str(&format!( + r#" + +"#, + problem_family + )); + // Get all problems in this family + let problems_in_family: Vec<_> = data + .iter() + .filter(|(problem, _)| { + report_generator::get_family(&problem.get_name()) == *problem_family + }) + .collect(); + if problems_in_family.is_empty() { + continue; + } + for optimizer_family in &optimizer_families { + let cell_data = + calculate_family_performance_data(&problems_in_family, optimizer_family)?; + // Collect scores for this row to find best/worst + let mut row_scores = Vec::new(); + for opt_fam in &optimizer_families { + let data = calculate_family_performance_data(&problems_in_family, opt_fam)?; + if data.average_ranking.is_finite() { + row_scores.push((opt_fam.clone(), data.average_ranking)); + } + } + // Find best and worst in this row + let best_family = row_scores + .iter() + .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(f, _)| f.as_str()); + let worst_family = row_scores + .iter() + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(f, _)| f.as_str()); + let cell_style = if cell_data.average_ranking.is_finite() { + if Some(optimizer_family.as_str()) == best_family { + "border: 1px solid #ddd; padding: 6px; text-align: center; background-color: #90EE90; font-size: 10px;" + } else if Some(optimizer_family.as_str()) == worst_family { + "border: 1px solid #ddd; padding: 6px; text-align: center; background-color: #FFB6C1; font-size: 10px;" + } else { + "border: 1px solid #ddd; padding: 6px; text-align: center; font-size: 10px;" + } + } else { + "border: 1px solid #ddd; padding: 6px; text-align: center; font-size: 10px;" + }; + content.push_str(&format!( + r#" +"#, + cell_style, + cell_data.average_ranking, + cell_data.best_rank_average, + cell_data.best_variant, + cell_data.worst_variant + )); + } + content.push_str("\n"); + } + content.push_str("
Problem Family{}
{} +
Avg Rank: {:.1}
+
Best Rank: {:.1}
+
Best Var: {}
+
Worst Var: {}
+
\n"); + content.push_str(r#"
+

Legend:

+
    +
  • Avg Rank: Average ranking of all variants in the optimizer family across problems in the problem family (lower is better)
    • +
  • Green cells indicate the best performing optimizer family for that problem family
    • +
  • Red cells indicate the worst performing optimizer family for that problem family
    • +
+
+"#); + Ok(()) + } + + /// Generate LaTeX content for the report + fn generate_latex( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut content = String::from( + r#"\documentclass{article} +\usepackage{booktabs} +\usepackage{array} +\usepackage{multirow} +\usepackage{xcolor} +\usepackage{siunitx} +\usepackage{adjustbox} +\usepackage{rotating} +\usepackage[margin=0.5in]{geometry} +\usepackage{longtable} +\definecolor{bestgreen}{RGB}{0,150,0} +\definecolor{worstred}{RGB}{200,0,0} +\begin{document} + +\title{Family vs Family Performance Matrix} +\author{QQN Optimizer Benchmark Suite} +\date{\today} +\maketitle + +"#, + ); + + let table_content = generate_family_vs_family_table_content(data)?; + content.push_str(&table_content); + content.push_str("\n\\end{document}"); + + Ok(content) + } + + /// Generate Markdown content for the report + fn generate_markdown( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut content = String::from( + r#"# Family vs Family Performance Matrix + +This report shows how different optimizer families perform across different problem families. + +"#, + ); + + // Generate a simplified markdown table since HTML tables in markdown are complex + let mut all_optimizer_families = HashSet::new(); + let mut all_problem_families = HashSet::new(); + + for (problem, results) in data { + let problem_family = report_generator::get_family(&problem.get_name()); + all_problem_families.insert(problem_family); + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + all_optimizer_families.insert(optimizer_family); + } + } + + let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); + let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); + optimizer_families.sort(); + problem_families.sort(); + + if optimizer_families.is_empty() || problem_families.is_empty() { + content.push_str("*No data available for family comparison.*\n\n"); + return Ok(content); + } + + // Create markdown table header + content.push_str("| Problem Family |"); + for optimizer_family in &optimizer_families { + content.push_str(&format!(" {} |", optimizer_family)); + } + content.push_str("\n|"); + content.push_str(&"---|".repeat(optimizer_families.len() + 1)); + content.push_str("\n"); + + // Generate table rows + for problem_family in &problem_families { + content.push_str(&format!("| **{}** |", problem_family)); + + let problems_in_family: Vec<_> = data + .iter() + .filter(|(problem, _)| { + report_generator::get_family(&problem.get_name()) == *problem_family + }) + .collect(); + + for optimizer_family in &optimizer_families { + let cell_data = + calculate_family_performance_data(&problems_in_family, optimizer_family)?; + + let cell_content = if cell_data.average_ranking.is_finite() { + format!( + " {:.1} / {:.1}
Best: {}
Worst: {} |", + cell_data.average_ranking, + cell_data.best_rank_average, + cell_data.best_variant, + cell_data.worst_variant + ) + } else { + " N/A |".to_string() + }; + content.push_str(&cell_content); + } + content.push_str("\n"); + } + + content.push_str( + r#" +## Legend + +- **First value**: Average ranking of all variants in the optimizer family (lower is better) +- **Second value**: Average of the best rank achieved by any variant in the optimizer family +- **Best**: The specific optimizer variant that achieved the best average rank +- **Worst**: The specific optimizer variant that achieved the worst average rank + +"#, + ); + + Ok(content) + } + + /// Generate CSV content for the report + fn generate_csv( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut content = String::from("Problem Family,Optimizer Family,Average Ranking,Best Rank Average,Best Variant,Worst Variant\n"); + + let mut all_optimizer_families = HashSet::new(); + let mut all_problem_families = HashSet::new(); + + for (problem, results) in data { + let problem_family = report_generator::get_family(&problem.get_name()); + all_problem_families.insert(problem_family); + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + all_optimizer_families.insert(optimizer_family); + } + } + + let mut optimizer_families: Vec<_> = all_optimizer_families.into_iter().collect(); + let mut problem_families: Vec<_> = all_problem_families.into_iter().collect(); + optimizer_families.sort(); + problem_families.sort(); + + for problem_family in &problem_families { + let problems_in_family: Vec<_> = data + .iter() + .filter(|(problem, _)| { + report_generator::get_family(&problem.get_name()) == *problem_family + }) + .collect(); + + for optimizer_family in &optimizer_families { + let cell_data = + calculate_family_performance_data(&problems_in_family, optimizer_family)?; + + content.push_str(&format!( + "{},{},{:.3},{:.3},{},{}\n", + problem_family, + optimizer_family, + if cell_data.average_ranking.is_finite() { + cell_data.average_ranking + } else { + f64::NAN + }, + if cell_data.best_rank_average.is_finite() { + cell_data.best_rank_average + } else { + f64::NAN + }, + cell_data.best_variant, + cell_data.worst_variant + )); + } + } + + Ok(content) + } +} + +impl Default for FamilyVsFamilyReport { + fn default() -> Self { + Self::new() + } +} + +impl Report for FamilyVsFamilyReport { + fn name(&self) -> &'static str { + "family_vs_family" + } + + fn description(&self) -> &'static str { + "Comparison matrix showing how different optimizer families perform across different problem families" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> Result<()> { + let content = self.generate_content(data, config)?; + std::fs::write(output_path, content)?; + Ok(()) + } + + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> Result<()> { + if data.is_empty() { + anyhow::bail!("No benchmark data provided"); + } + + // Check that we have results with optimizer names + let has_results = data.iter().any(|(_, results)| !results.results.is_empty()); + if !has_results { + anyhow::bail!("No benchmark results found in data"); + } + + // Check that we have at least one optimizer family and problem family + let mut optimizer_families = HashSet::new(); + let mut problem_families = HashSet::new(); + + for (problem, results) in data { + let problem_family = report_generator::get_family(&problem.get_name()); + problem_families.insert(problem_family); + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + optimizer_families.insert(optimizer_family); + } + } + + if optimizer_families.is_empty() { + anyhow::bail!("No optimizer families found in data"); + } + + if problem_families.is_empty() { + anyhow::bail!("No problem families found in data"); + } + + Ok(()) + } + + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let mut optimizer_families = HashSet::new(); + let mut problem_families = HashSet::new(); + let mut total_data_points = 0; + + for (problem, results) in data { + let problem_family = report_generator::get_family(&problem.get_name()); + problem_families.insert(problem_family); + total_data_points += results.results.len(); + + for result in &results.results { + let optimizer_family = get_optimizer_family(&result.optimizer_name); + optimizer_families.insert(optimizer_family); + } + } + + ReportMetadata { + report_type: self.name().to_string(), + generated_at: Default::default(), + problem_count: problem_families.len(), + optimizer_count: optimizer_families.len(), + data_points: total_data_points, + } + } + + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::unified_report_tests::UnifiedReportTestSuite; + + #[test] + fn test_family_vs_family_report_with_unified_suite() { + let report = FamilyVsFamilyReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + + #[test] + fn test_family_vs_family_report_basic_functionality() { + let report = FamilyVsFamilyReport::new(); + assert_eq!(report.name(), "family_vs_family"); + assert!(!report.description().is_empty()); + assert_eq!(report.supported_formats().len(), 4); + } + + #[test] + fn test_family_vs_family_report_content_generation() { + let report = FamilyVsFamilyReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + for format in &report.supported_formats() { + let config = ReportConfig { + format: format.clone(), + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!( + !content.is_empty(), + "Content should not be empty for format {:?}", + format + ); + + // Format-specific validations + match format { + ReportFormat::Html => { + assert!( + content.contains(""), + "HTML content should contain html tags. Content: {}", + content + ); + assert!(content.contains("Family vs Family")); + } + ReportFormat::Latex => { + assert!(content.contains("\\documentclass")); + assert!(content.contains("\\begin{document}")); + assert!(content.contains("\\end{document}")); + } + ReportFormat::Markdown => { + assert!(content.contains("# Family vs Family")); + assert!(content.contains("|")); + } + ReportFormat::Csv => { + assert!(content.contains("Problem Family,Optimizer Family")); + assert!(content.contains(",")); + } + } + } + } + + #[test] + fn test_family_vs_family_report_validation() { + let report = FamilyVsFamilyReport::new(); + + // Test empty data + let empty_data = vec![]; + assert!(report.validate_data(&empty_data).is_err()); + + // Test valid data + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + assert!(report.validate_data(&data_refs).is_ok()); + + // Test data with empty results + let empty_results_data = UnifiedReportTestSuite::create_empty_results_data(); + let empty_refs: Vec<_> = empty_results_data + .iter() + .map(|(p, r)| (p, r.clone())) + .collect(); + assert!(report.validate_data(&empty_refs).is_err()); + } + + #[test] + fn test_family_vs_family_report_metadata() { + let report = FamilyVsFamilyReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let metadata = report.get_metadata(&data_refs); + assert_eq!(metadata.report_type, "family_vs_family"); + assert!(metadata.problem_count > 0); + assert!(metadata.optimizer_count > 0); + assert!(metadata.data_points > 0); + } +} diff --git a/src/experiment_runner/reports/heatmap.rs b/src/experiment_runner/reports/heatmap.rs index 276c250b..940faf4a 100644 --- a/src/experiment_runner/reports/heatmap.rs +++ b/src/experiment_runner/reports/heatmap.rs @@ -1,9 +1,378 @@ use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; -use crate::experiment_runner::report_generator; +use crate::experiment_runner::{ + report_generator, Report, ReportConfig, ReportFormat, ReportMetadata, +}; use anyhow::Context; +use html_escape::encode_text; +use std::collections::HashMap; use std::fs; use std::path::Path; +/// Success Rate Heatmap Report +pub struct SuccessRateHeatmapReport; +impl SuccessRateHeatmapReport { + pub fn new() -> Self { + Self + } +} +impl Report for SuccessRateHeatmapReport { + fn name(&self) -> &'static str { + "success_rate_heatmap" + } + fn description(&self) -> &'static str { + "Color-coded heatmap showing success rates across optimizer-problem combinations" + } + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> anyhow::Result<()> { + let content = self.generate_content(data, config)?; + fs::write(output_path, content).with_context(|| { + format!( + "Failed to write success rate heatmap report to: {}", + output_path.display() + ) + })?; + Ok(()) + } + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> anyhow::Result<()> { + if data.is_empty() { + return Err(anyhow::anyhow!("No benchmark data provided")); + } + // Check that we have at least one optimizer result + let has_results = data.iter().any(|(_, results)| !results.results.is_empty()); + if !has_results { + return Err(anyhow::anyhow!( + "No optimizer results found in benchmark data" + )); + } + Ok(()) + } + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let mut all_optimizers = std::collections::HashSet::new(); + let problem_count = data.len(); + for (_, results) in data { + for result in &results.results { + all_optimizers.insert(result.optimizer_name.clone()); + } + } + let mut metadata = HashMap::new(); + metadata.insert( + "optimizer_count".to_string(), + all_optimizers.len().to_string(), + ); + metadata.insert("problem_count".to_string(), problem_count.to_string()); + metadata.insert("report_type".to_string(), "heatmap".to_string()); + ReportMetadata { + report_type: "success_rate_heatmap".to_string(), + generated_at: Default::default(), + problem_count, + optimizer_count: 10, + data_points: 10, + } + } + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} +impl SuccessRateHeatmapReport { + fn generate_html( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizers, _) = self.collect_optimizers_and_problems(data); + if optimizers.is_empty() { + return Ok("

No data available for heatmap generation.

".to_string()); + } + let mut html = String::from( + r#" + + + Success Rate Heatmap + + + +

Success Rate Heatmap

+

Color-coded success rates across all optimizer-problem combinations

+ + + + "#, + ); + for optimizer in &optimizers { + html.push_str(&format!("", encode_text(optimizer))); + } + html.push_str(""); + for (problem, results) in data { + let problem_name = problem.get_name(); + html.push_str(&format!( + "", + encode_text(&problem_name) + )); + for optimizer in &optimizers { + let (success_rate, has_data) = self.calculate_success_rate(results, optimizer); + let (class, display_text) = self.get_html_cell_style(success_rate, has_data); + html.push_str(&format!("", class, display_text)); + } + html.push_str(""); + } + html.push_str( + r#"
Problem{}
{}{}
+
+ Legend: + 90-100% Excellent + 50-89% Good + 10-49% Poor + 0-9% Very Poor + N/A No Data +
+ +"#, + ); + Ok(html) + } + fn generate_latex( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizers, _) = self.collect_optimizers_and_problems(data); + if optimizers.is_empty() { + return Ok(String::new()); + } + let col_spec = format!("l{}", "c".repeat(optimizers.len())); + let mut latex_content = String::from( + r#"\documentclass{article} +\usepackage{booktabs} +\usepackage{array} +\usepackage[table]{xcolor} +\usepackage{adjustbox} +\usepackage{rotating} +\usepackage[margin=1in]{geometry} +\begin{document} +"#, + ); + latex_content.push_str(&format!( + r#"\begin{{table}}[htbp] +\centering +\caption{{Success Rate Heatmap: Color-coded Success Rates Across All Optimizer-Problem Combinations}} +\label{{tab:success_rate_heatmap}} +\adjustbox{{width=\textwidth,center}}{{ +\begin{{tabular}}{{{}}} +\toprule +\textbf{{Problem}} {}\\ +\midrule +"#, + col_spec, + optimizers + .iter() + .map(|opt| format!("& \\rotatebox{{90}}{{\\textbf{{{}}}}}", report_generator::escape_latex(opt))) + .collect::>() + .join(" ") + )); + for (problem, results) in data { + let problem_name = problem.get_name(); + latex_content.push_str(&format!( + "\\textbf{{{}}} ", + report_generator::escape_latex(&problem_name) + )); + for optimizer in &optimizers { + let (success_rate, has_data) = self.calculate_success_rate(results, optimizer); + let cell_content = self.get_latex_cell_content(success_rate, has_data); + latex_content.push_str(&cell_content); + } + latex_content.push_str(" \\\\\n"); + } + latex_content.push_str( + r#"\bottomrule +\end{tabular} +} +\end{table} +\textbf{Legend:} +\colorbox{green!70}{90-100\%} Excellent, +\colorbox{yellow!70}{50-89\%} Good, +\colorbox{orange!70}{10-49\%} Poor, +\colorbox{red!70}{\textcolor{white}{0-9\%}} Very Poor, +\colorbox{gray!30}{\textcolor{white}{N/A}} No Data. +Quickly identifies which optimizers work on which problem types. +\end{document} +"#, + ); + Ok(latex_content) + } + fn generate_markdown( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizers, _) = self.collect_optimizers_and_problems(data); + if optimizers.is_empty() { + return Ok("No data available for heatmap generation.".to_string()); + } + let mut markdown = String::from("# Success Rate Heatmap\n\nColor-coded success rates across all optimizer-problem combinations\n\n"); + // Table header + markdown.push_str("| Problem |"); + for optimizer in &optimizers { + markdown.push_str(&format!(" {} |", optimizer)); + } + markdown.push('\n'); + // Table separator + markdown.push_str("|---------|"); + for _ in &optimizers { + markdown.push_str("---------|"); + } + markdown.push('\n'); + // Table rows + for (problem, results) in data { + let problem_name = problem.get_name(); + markdown.push_str(&format!("| **{}** |", problem_name)); + for optimizer in &optimizers { + let (success_rate, has_data) = self.calculate_success_rate(results, optimizer); + let display_text = if has_data { + format!("{:.0}%", success_rate) + } else { + "N/A".to_string() + }; + markdown.push_str(&format!(" {} |", display_text)); + } + markdown.push('\n'); + } + markdown.push_str("\n**Legend:**\n"); + markdown.push_str("- 90-100%: Excellent\n"); + markdown.push_str("- 50-89%: Good\n"); + markdown.push_str("- 10-49%: Poor\n"); + markdown.push_str("- 0-9%: Very Poor\n"); + markdown.push_str("- N/A: No Data\n"); + Ok(markdown) + } + fn generate_csv( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let (optimizers, _) = self.collect_optimizers_and_problems(data); + if optimizers.is_empty() { + return Ok( + "Problem,Message\nNo Data,No data available for heatmap generation".to_string(), + ); + } + let mut csv = String::from("Problem"); + for optimizer in &optimizers { + csv.push_str(&format!(",{}", optimizer)); + } + csv.push('\n'); + for (problem, results) in data { + let problem_name = problem.get_name(); + csv.push_str(&problem_name); + for optimizer in &optimizers { + let (success_rate, has_data) = self.calculate_success_rate(results, optimizer); + let value = if has_data { + format!("{:.1}", success_rate) + } else { + "N/A".to_string() + }; + csv.push_str(&format!(",{}", value)); + } + csv.push('\n'); + } + Ok(csv) + } + fn collect_optimizers_and_problems( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + ) -> (Vec, Vec) { + let mut all_optimizers = std::collections::HashSet::new(); + let mut all_problems = Vec::new(); + for (problem, results) in data { + all_problems.push(problem.get_name()); + for result in &results.results { + all_optimizers.insert(result.optimizer_name.clone()); + } + } + let mut optimizers: Vec<_> = all_optimizers.into_iter().collect(); + optimizers.sort(); + (optimizers, all_problems) + } + fn calculate_success_rate(&self, results: &BenchmarkResults, optimizer: &str) -> (f64, bool) { + let optimizer_results: Vec<_> = results + .results + .iter() + .filter(|r| r.optimizer_name == optimizer) + .collect(); + if optimizer_results.is_empty() { + (0.0, false) + } else { + let successful = optimizer_results + .iter() + .filter(|r| r.convergence_achieved) + .count(); + let success_rate = successful as f64 / optimizer_results.len() as f64 * 100.0; + (success_rate, true) + } + } + fn get_html_cell_style(&self, success_rate: f64, has_data: bool) -> (&'static str, String) { + if !has_data { + ("no-data", "N/A".to_string()) + } else if success_rate >= 90.0 { + ("excellent", format!("{:.0}%", success_rate)) + } else if success_rate >= 50.0 { + ("good", format!("{:.0}%", success_rate)) + } else if success_rate >= 10.0 { + ("poor", format!("{:.0}%", success_rate)) + } else { + ("very-poor", format!("{:.0}%", success_rate)) + } + } + fn get_latex_cell_content(&self, success_rate: f64, has_data: bool) -> String { + if !has_data { + "& \\cellcolor{gray!30}\\textcolor{white}{N/A}".to_string() + } else { + let (color, text_color) = if success_rate >= 90.0 { + ("green!70", "black") + } else if success_rate >= 50.0 { + ("yellow!70", "black") + } else if success_rate >= 10.0 { + ("orange!70", "black") + } else { + ("red!70", "white") + }; + format!("& \\cellcolor{{{color}}}\\textcolor{{{text_color}}}{{{success_rate:.0}\\%}}") + } + } +} + /// Generate success rate heatmap table content (without document wrapper) pub fn generate_success_rate_heatmap_table_content( all_results: &[(&ProblemSpec, BenchmarkResults)], @@ -223,3 +592,13 @@ Quickly identifies which optimizers work on which problem types. ); Ok(()) } +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::UnifiedReportTestSuite; + #[test] + fn test_success_rate_heatmap_report() { + let report = SuccessRateHeatmapReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } +} diff --git a/src/experiment_runner/reports/mod.rs b/src/experiment_runner/reports/mod.rs index 23133e8c..b565c6e4 100644 --- a/src/experiment_runner/reports/mod.rs +++ b/src/experiment_runner/reports/mod.rs @@ -2,7 +2,10 @@ pub mod comparison_matrix; pub mod convergence_analysis; pub mod efficiency_matrix; pub mod family_vs_family; +pub mod family_vs_family_report; pub mod heatmap; pub mod performance_analysis; pub mod performance_table; pub mod summary_statistics; +pub mod unified_performance_table; +pub mod unified_summary_statistics; diff --git a/src/experiment_runner/reports/performance_analysis.rs b/src/experiment_runner/reports/performance_analysis.rs index f111a8e0..eb1a4cf1 100644 --- a/src/experiment_runner/reports/performance_analysis.rs +++ b/src/experiment_runner/reports/performance_analysis.rs @@ -1,17 +1,29 @@ -use crate::benchmarks::evaluation::SingleResult; +use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec, SingleResult}; +use crate::experiment_runner::{Report, ReportConfig, ReportFormat, ReportMetadata}; +use anyhow::Result; +use std::collections::HashMap; +use std::path::Path; -pub fn generate_performance_analysis(runs: &[&SingleResult]) -> anyhow::Result { - let mut content = String::from("## Performance Analysis\n\n"); - let total_func_evals: usize = runs.iter().map(|r| r.function_evaluations).sum(); - let total_grad_evals: usize = runs.iter().map(|r| r.gradient_evaluations).sum(); - let total_time: f64 = runs.iter().map(|r| r.execution_time.as_secs_f64()).sum(); - let total_iterations: usize = runs.iter().map(|r| r.iterations).sum(); - let avg_func_evals = total_func_evals as f64 / runs.len() as f64; - let avg_grad_evals = total_grad_evals as f64 / runs.len() as f64; - let avg_time = total_time / runs.len() as f64; - let avg_iterations = total_iterations as f64 / runs.len() as f64; - content.push_str(&format!( - r#"### Computational Efficiency +pub struct PerformanceAnalysisReport; + +impl PerformanceAnalysisReport { + pub fn new() -> Self { + Self + } + + fn generate_analysis_content(&self, runs: &[&SingleResult]) -> String { + let mut content = String::from("## Performance Analysis\n\n"); + let total_func_evals: usize = runs.iter().map(|r| r.function_evaluations).sum(); + let total_grad_evals: usize = runs.iter().map(|r| r.gradient_evaluations).sum(); + let total_time: f64 = runs.iter().map(|r| r.execution_time.as_secs_f64()).sum(); + let total_iterations: usize = runs.iter().map(|r| r.iterations).sum(); + let avg_func_evals = total_func_evals as f64 / runs.len() as f64; + let avg_grad_evals = total_grad_evals as f64 / runs.len() as f64; + let avg_time = total_time / runs.len() as f64; + let avg_iterations = total_iterations as f64 / runs.len() as f64; + + content.push_str(&format!( + r#"### Computational Efficiency - **Average Function Evaluations per Run:** {:.1} - **Average Gradient Evaluations per Run:** {:.1} - **Average Iterations per Run:** {:.1} @@ -24,28 +36,375 @@ pub fn generate_performance_analysis(runs: &[&SingleResult]) -> anyhow::Result 0.0 { - avg_func_evals / avg_time - } else { - 0.0 - }, - if avg_time > 0.0 { - avg_iterations / avg_time - } else { - 0.0 - }, - total_func_evals, - total_grad_evals, - total_time, - if total_grad_evals > 0 { - total_func_evals as f64 / total_grad_evals as f64 - } else { - 0.0 - } - )); - Ok(content) + avg_func_evals, + avg_grad_evals, + avg_iterations, + avg_time, + if avg_time > 0.0 { + avg_func_evals / avg_time + } else { + 0.0 + }, + if avg_time > 0.0 { + avg_iterations / avg_time + } else { + 0.0 + }, + total_func_evals, + total_grad_evals, + total_time, + if total_grad_evals > 0 { + total_func_evals as f64 / total_grad_evals as f64 + } else { + 0.0 + } + )); + content + } + + fn generate_html( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + let mut html = String::from( + r#" + + + Performance Analysis Report + + + +"#, + ); + + for (problem_spec, results) in data { + html.push_str(&format!( + r#"
+

Problem: {}

+"#, + problem_spec.name.as_deref().unwrap_or("Unknown") + )); + + // Group results by optimizer + let mut optimizer_results: std::collections::HashMap> = + std::collections::HashMap::new(); + for result in &results.results { + optimizer_results + .entry(result.optimizer_name.clone()) + .or_insert_with(Vec::new) + .push(result); + } + + for (optimizer_name, runs) in optimizer_results { + html.push_str(&format!("

Optimizer: {}

\n", optimizer_name)); + let analysis = self.generate_analysis_content(&runs); + // Convert markdown to basic HTML + let html_analysis = analysis + .replace("## ", "

") + .replace("### ", "

") + .replace("- **", "
  • ") + .replace(":**", ":") + .replace("\n\n", "\n\n
      \n") + .replace("\n", "\n"); + html.push_str(&format!("
        \n{}\n
      \n", html_analysis)); + } + html.push_str("

    • \n"); + } + + html.push_str("\n"); + Ok(html) + } + + fn generate_latex( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + let mut latex = String::from( + r#"\documentclass{article} +\usepackage[utf8]{inputenc} +\usepackage{amsmath} +\usepackage{booktabs} +\usepackage{geometry} +\geometry{margin=1in} +\title{Performance Analysis Report} +\date{\today} +\begin{document} +\maketitle + +"#, + ); + + for (problem_spec, results) in data { + latex.push_str(&format!( + "\\section{{Problem: {}}}\n\n", + problem_spec.name.as_deref().unwrap_or("Unknown") + )); + + // Group results by optimizer + let mut optimizer_results: std::collections::HashMap> = + std::collections::HashMap::new(); + for result in &results.results { + optimizer_results + .entry(result.optimizer_name.clone()) + .or_insert_with(Vec::new) + .push(result); + } + + for (optimizer_name, runs) in optimizer_results { + latex.push_str(&format!( + "\\subsection{{Optimizer: {}}}\n\n", + optimizer_name + )); + let analysis = self.generate_analysis_content(&runs); + // Convert markdown to LaTeX + let latex_analysis = analysis + .replace("## ", "\\section{") + .replace("### ", "\\subsection{") + .replace("- **", "\\item \\textbf{") + .replace(":**", ":}") + .replace("\n\n", "}\n\n\\begin{itemize}\n") + .replace("\n", "}\n"); + latex.push_str(&format!( + "\\begin{{itemize}}\n{}\\end{{itemize}}\n\n", + latex_analysis + )); + } + } + + latex.push_str("\\end{document}"); + Ok(latex) + } + + fn generate_markdown( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + let mut markdown = String::from("# Performance Analysis Report\n\n"); + + for (problem_spec, results) in data { + markdown.push_str(&format!( + "## Problem: {}\n\n", + problem_spec.name.as_deref().unwrap_or("Unknown") + )); + + // Group results by optimizer + let mut optimizer_results: std::collections::HashMap> = + std::collections::HashMap::new(); + for result in &results.results { + optimizer_results + .entry(result.optimizer_name.clone()) + .or_insert_with(Vec::new) + .push(result); + } + + for (optimizer_name, runs) in optimizer_results { + markdown.push_str(&format!("### Optimizer: {}\n\n", optimizer_name)); + let analysis = self.generate_analysis_content(&runs); + markdown.push_str(&analysis); + markdown.push_str("\n\n"); + } + } + + Ok(markdown) + } + + fn generate_csv( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + let mut csv = String::from("Problem,Optimizer,Avg_Function_Evals,Avg_Gradient_Evals,Avg_Iterations,Avg_Time_Sec,Total_Function_Evals,Total_Gradient_Evals,Total_Time_Sec,Function_Gradient_Ratio\n"); + + for (problem_spec, results) in data { + // Group results by optimizer + let mut optimizer_results: std::collections::HashMap> = + std::collections::HashMap::new(); + for result in &results.results { + optimizer_results + .entry(result.optimizer_name.clone()) + .or_insert_with(Vec::new) + .push(result); + } + + for (optimizer_name, runs) in optimizer_results { + let total_func_evals: usize = runs.iter().map(|r| r.function_evaluations).sum(); + let total_grad_evals: usize = runs.iter().map(|r| r.gradient_evaluations).sum(); + let total_time: f64 = runs.iter().map(|r| r.execution_time.as_secs_f64()).sum(); + let total_iterations: usize = runs.iter().map(|r| r.iterations).sum(); + let avg_func_evals = total_func_evals as f64 / runs.len() as f64; + let avg_grad_evals = total_grad_evals as f64 / runs.len() as f64; + let avg_time = total_time / runs.len() as f64; + let avg_iterations = total_iterations as f64 / runs.len() as f64; + let func_grad_ratio = if total_grad_evals > 0 { + total_func_evals as f64 / total_grad_evals as f64 + } else { + 0.0 + }; + + csv.push_str(&format!( + "{},{},{:.1},{:.1},{:.1},{:.3},{},{},{:.1},{:.2}\n", + problem_spec.name.as_deref().unwrap_or("Unknown"), + &optimizer_name, + avg_func_evals, + avg_grad_evals, + avg_iterations, + avg_time, + total_func_evals, + total_grad_evals, + total_time, + func_grad_ratio + )); + } + } + + Ok(csv) + } +} + +impl Report for PerformanceAnalysisReport { + fn name(&self) -> &'static str { + "performance_analysis" + } + + fn description(&self) -> &'static str { + "Detailed performance analysis including computational efficiency and resource utilization metrics" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> Result<()> { + let content = self.generate_content(data, config)?; + std::fs::write(output_path, content)?; + Ok(()) + } + + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> Result<()> { + if data.is_empty() { + return Err(anyhow::anyhow!("No benchmark data provided")); + } + + for (problem_spec, results) in data { + if problem_spec.name.is_none() { + return Err(anyhow::anyhow!("Problem spec has empty name")); + } + if results.results.is_empty() { + return Err(anyhow::anyhow!( + "No results for problem: {}", + problem_spec.name.as_deref().unwrap_or("Unknown") + )); + } + } + Ok(()) + } + + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let total_problems = data.len(); + let total_optimizers: usize = data + .iter() + .flat_map(|(_, results)| &results.results) + .map(|r| &r.optimizer_name) + .collect::>() + .len(); + let total_runs: usize = data.iter().map(|(_, results)| results.results.len()).sum(); + + let mut metadata = HashMap::new(); + metadata.insert("total_problems".to_string(), total_problems.to_string()); + metadata.insert("total_optimizers".to_string(), total_optimizers.to_string()); + metadata.insert("total_runs".to_string(), total_runs.to_string()); + metadata.insert( + "report_type".to_string(), + "performance_analysis".to_string(), + ); + + ReportMetadata { + report_type: "performance_analysis".to_string(), + generated_at: chrono::Utc::now(), + problem_count: total_problems, + optimizer_count: total_optimizers, + data_points: total_runs, + } + } + + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} + +// Legacy function for backward compatibility +pub fn generate_performance_analysis(runs: &[&SingleResult]) -> anyhow::Result { + let report = PerformanceAnalysisReport::new(); + Ok(report.generate_analysis_content(runs)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::UnifiedReportTestSuite; + + #[test] + fn test_performance_analysis_report() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + + #[test] + fn test_basic_functionality() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_basic_functionality(&report).unwrap(); + } + + #[test] + fn test_content_generation() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_content_generation(&report).unwrap(); + } + + #[test] + fn test_data_validation() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_data_validation(&report).unwrap(); + } + + #[test] + fn test_metadata_generation() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_metadata_generation(&report).unwrap(); + } + + #[test] + fn test_file_export() { + let report = PerformanceAnalysisReport::new(); + UnifiedReportTestSuite::test_file_export(&report).unwrap(); + } } diff --git a/src/experiment_runner/reports/performance_table.rs b/src/experiment_runner/reports/performance_table.rs index c1cc5f86..38759f3a 100644 --- a/src/experiment_runner/reports/performance_table.rs +++ b/src/experiment_runner/reports/performance_table.rs @@ -1,15 +1,432 @@ use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; -use crate::experiment_runner::report_generator; +use crate::experiment_runner::{ + report_generator, Report, ReportConfig, ReportFormat, ReportMetadata, +}; use anyhow::Context; use std::collections::HashMap; use std::fs; use std::path::Path; +/// Performance table report showing detailed metrics for each optimizer-problem combination +pub struct PerformanceTableReport; +impl PerformanceTableReport { + pub fn new() -> Self { + Self + } + fn calculate_performance_data( + &self, + all_results: &[(&ProblemSpec, BenchmarkResults)], + ) -> Vec<(String, Vec<(String, f64, f64, f64, f64, f64, f64, f64)>)> { + let mut problem_data = Vec::new(); + for (problem, results) in all_results { + let problem_name = problem.get_name(); + let mut optimizer_stats = HashMap::new(); + for result in &results.results { + let stats = optimizer_stats + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + stats.push(result); + } + let mut perf_data = Vec::new(); + for (optimizer, runs) in &optimizer_stats { + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + if final_values.is_empty() { + continue; + } + let function_evals: Vec = + runs.iter().map(|r| r.function_evaluations as f64).collect(); + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let execution_times: Vec = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .collect(); + let mean_final = final_values.iter().sum::() / final_values.len() as f64; + let std_final = { + let variance = final_values + .iter() + .map(|x| (x - mean_final).powi(2)) + .sum::() + / final_values.len() as f64; + variance.sqrt() + }; + let best_final = final_values.iter().cloned().fold(f64::INFINITY, f64::min); + let worst_final = final_values + .iter() + .cloned() + .fold(f64::NEG_INFINITY, f64::max); + let mean_function_evals = + function_evals.iter().sum::() / function_evals.len() as f64; + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + let mean_time = execution_times.iter().sum::() / execution_times.len() as f64; + perf_data.push(( + optimizer.clone(), + mean_final, + std_final, + best_final, + worst_final, + mean_function_evals, + success_rate, + mean_time, + )); + } + // Sort by success rate first, then by mean final value + perf_data.sort_by(|a, b| { + let success_cmp = b.6.partial_cmp(&a.6).unwrap_or(std::cmp::Ordering::Equal); + if success_cmp != std::cmp::Ordering::Equal { + success_cmp + } else { + a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal) + } + }); + problem_data.push((problem_name, perf_data)); + } + problem_data + } + fn generate_html( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let performance_data = self.calculate_performance_data(data); + let mut html = String::from( + r#" + + + Performance Table Report + + + +

    Performance Table Report

    +

    Detailed performance metrics for each optimizer-problem combination.

    + + + + + + + + + + + + + + + +"#, + ); + for (problem_name, perf_data) in performance_data { + for ( + i, + ( + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time, + ), + ) in perf_data.iter().enumerate() + { + let problem_cell = if i == 0 { + format!( + "", + perf_data.len(), + problem_name + ) + } else { + String::new() + }; + let row_class = if i == 0 { "best-optimizer" } else { "" }; + html.push_str(&format!( + "{} + + + + + + + + + \n", + row_class, + problem_cell, + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time + )); + } + } + html.push_str("
    ProblemOptimizerMean Final ValueStd DevBest ValueWorst ValueMean Func EvalsSuccess Rate (%)Mean Time (s)
    {}
    {}{:.2e}{:.2e}{:.2e}{:.2e}{:.1}{:.1}{:.3}
    "); + Ok(html) + } + fn generate_latex( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let performance_data = self.calculate_performance_data(data); + let mut latex_content = String::from( + r#"\documentclass{article} +\usepackage[margin=0.5in]{geometry} +\usepackage{booktabs} +\usepackage{array} +\usepackage{multirow} +\usepackage{longtable} +\usepackage{colortbl} +\usepackage{xcolor} +\usepackage{siunitx} +\usepackage{adjustbox} +\usepackage{rotating} +\usepackage{graphicx} +\begin{document} +\tiny +\begin{adjustbox}{width=\textwidth,center} +\begin{longtable}{p{2cm}p{2cm}p{1.2cm}p{1.2cm}p{1.2cm}p{1.2cm}p{1.2cm}p{1.2cm}p{1.2cm}} +\caption{Comprehensive Performance Comparison of Optimization Algorithms} \\ +\toprule +\textbf{Problem} & \textbf{Optimizer} & \textbf{Mean Final} & \textbf{Std Dev} & \textbf{Best} & \textbf{Worst} & \textbf{Mean Func} & \textbf{Success} & \textbf{Mean Time} \\ + & & \textbf{Value} & & \textbf{Value} & \textbf{Value} & \textbf{Evals} & \textbf{Rate (\%)} & \textbf{(s)} \\ +\midrule +\endfirsthead +\multicolumn{9}{c}% +{{\bfseries \tablename\ \thetable{} -- continued from previous page}} \\ +\toprule +\textbf{Problem} & \textbf{Optimizer} & \textbf{Mean Final} & \textbf{Std Dev} & \textbf{Best} & \textbf{Worst} & \textbf{Mean Func} & \textbf{Success} & \textbf{Mean Time} \\ + & & \textbf{Value} & & \textbf{Value} & \textbf{Value} & \textbf{Evals} & \textbf{Rate (\%)} & \textbf{(s)} \\ +\midrule +\endhead +\midrule \multicolumn{9}{r}{{Continued on next page}} \\ \midrule +\endfoot +\bottomrule +\endlastfoot +"#, + ); + for (problem_name, perf_data) in performance_data { + for ( + i, + ( + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time, + ), + ) in perf_data.iter().enumerate() + { + let problem_cell = if i == 0 { + format!( + "\\multirow{{{}}}{{*}}{{{}}}", + perf_data.len(), + report_generator::escape_latex(&problem_name) + ) + } else { + String::new() + }; + let optimizer_style = if i == 0 { + format!("\\textbf{{{}}}", report_generator::escape_latex(optimizer)) + } else { + report_generator::escape_latex(optimizer) + }; + latex_content.push_str(&format!( + "{problem_cell} & {optimizer_style} & {mean_final:.2e} & {std_final:.2e} & {best_final:.2e} & {worst_final:.2e} & {mean_func_evals:.1} & {success_rate:.1} & {mean_time:.3} \\\\\n" + )); + } + if !perf_data.is_empty() { + latex_content.push_str("\\midrule\n"); + } + } + latex_content.push_str( + r#"\end{longtable} +\end{adjustbox} +\end{document} +"#, + ); + Ok(latex_content) + } + fn generate_markdown( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let performance_data = self.calculate_performance_data(data); + let mut markdown = String::from("# Performance Table Report\n\n"); + markdown + .push_str("Detailed performance metrics for each optimizer-problem combination.\n\n"); + for (problem_name, perf_data) in performance_data { + markdown.push_str(&format!("## {}\n\n", problem_name)); + markdown.push_str("| Optimizer | Mean Final Value | Std Dev | Best Value | Worst Value | Mean Func Evals | Success Rate (%) | Mean Time (s) |\n"); + markdown.push_str("|-----------|------------------|---------|------------|-------------|-----------------|------------------|---------------|\n"); + for ( + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time, + ) in perf_data + { + markdown.push_str(&format!( + "| {} | {:.2e} | {:.2e} | {:.2e} | {:.2e} | {:.1} | {:.1} | {:.3} |\n", + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time + )); + } + markdown.push_str("\n"); + } + Ok(markdown) + } + fn generate_csv( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> anyhow::Result { + let performance_data = self.calculate_performance_data(data); + let mut csv = String::from("Problem,Optimizer,Mean Final Value,Std Dev,Best Value,Worst Value,Mean Func Evals,Success Rate (%),Mean Time (s)\n"); + for (problem_name, perf_data) in performance_data { + for ( + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time, + ) in perf_data + { + csv.push_str(&format!( + "{},{},{:.2e},{:.2e},{:.2e},{:.2e},{:.1},{:.1},{:.3}\n", + problem_name, + optimizer, + mean_final, + std_final, + best_final, + worst_final, + mean_func_evals, + success_rate, + mean_time + )); + } + } + Ok(csv) + } +} +impl Report for PerformanceTableReport { + fn name(&self) -> &'static str { + "performance_table" + } + fn description(&self) -> &'static str { + "Shows detailed performance metrics for each optimizer-problem combination" + } + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> anyhow::Result { + match config.format { + ReportFormat::Html => self.generate_html(data, config), + ReportFormat::Latex => self.generate_latex(data, config), + ReportFormat::Markdown => self.generate_markdown(data, config), + ReportFormat::Csv => self.generate_csv(data, config), + } + } + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> anyhow::Result<()> { + let content = self.generate_content(data, config)?; + fs::write(output_path, content).with_context(|| { + format!( + "Failed to write performance table report to: {}", + output_path.display() + ) + })?; + Ok(()) + } + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> anyhow::Result<()> { + if data.is_empty() { + return Err(anyhow::anyhow!("No benchmark data provided")); + } + for (problem, results) in data { + if results.results.is_empty() { + return Err(anyhow::anyhow!( + "No results for problem: {}", + problem.get_name() + )); + } + } + Ok(()) + } + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let total_problems = data.len(); + let total_optimizers: std::collections::HashSet = data + .iter() + .flat_map(|(_, results)| results.results.iter().map(|r| r.optimizer_name.clone())) + .collect(); + let total_runs: usize = data.iter().map(|(_, results)| results.results.len()).sum(); + ReportMetadata { + report_type: "performance_table".to_string(), + generated_at: Default::default(), + problem_count: total_problems, + optimizer_count: total_optimizers.len(), + data_points: total_runs, + } + } + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] + } +} +// Legacy functions for backward compatibility /// Generate main performance LaTeX table pub fn generate_main_performance_latex_table( all_results: &[(&ProblemSpec, BenchmarkResults)], latex_dir: &Path, ) -> anyhow::Result<()> { + let report = PerformanceTableReport::new(); + let config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + let content = report.generate_content(all_results, &config)?; + let mut latex_content = String::from( r#"\documentclass{article} \usepackage[margin=0.5in]{geometry} @@ -166,6 +583,13 @@ pub fn generate_main_performance_latex_table( pub fn generate_main_performance_table_content( all_results: &[(&ProblemSpec, BenchmarkResults)], ) -> anyhow::Result { + let report = PerformanceTableReport::new(); + let config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + let latex_content = report.generate_latex(all_results, &config)?; + let mut content = String::from( r#"\tiny \begin{adjustbox}{width=\textwidth,center} @@ -295,3 +719,31 @@ pub fn generate_main_performance_table_content( content.push_str("\\end{adjustbox}\n"); Ok(content) } +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::UnifiedReportTestSuite; + #[test] + fn test_performance_table_report() { + let report = PerformanceTableReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + #[test] + fn test_performance_table_basic_functionality() { + let report = PerformanceTableReport::new(); + assert_eq!(report.name(), "performance_table"); + assert_eq!( + report.description(), + "Shows detailed performance metrics for each optimizer-problem combination" + ); + assert_eq!( + report.supported_formats(), + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv + ] + ); + } +} diff --git a/src/experiment_runner/reports/unified_performance_table.rs b/src/experiment_runner/reports/unified_performance_table.rs new file mode 100644 index 00000000..9376f7dc --- /dev/null +++ b/src/experiment_runner/reports/unified_performance_table.rs @@ -0,0 +1,467 @@ +//! Performance table report implementation using the unified report trait. + +use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; +use crate::experiment_runner::unified_report::{Report, ReportConfig, ReportFormat}; +use anyhow::Result; +use std::collections::HashMap; + +/// Performance table report that provides detailed performance metrics +/// for each optimizer-problem combination. +pub struct PerformanceTableReport; + +impl PerformanceTableReport { + /// Create a new performance table report + pub fn new() -> Self { + Self + } +} + +impl Default for PerformanceTableReport { + fn default() -> Self { + Self::new() + } +} + +impl Report for PerformanceTableReport { + fn name(&self) -> &'static str { + "performance_table" + } + + fn description(&self) -> &'static str { + "Detailed performance table showing metrics for each optimizer-problem combination" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + self.validate_data(data)?; + + match config.format { + ReportFormat::Html => self.generate_html_content(data, config), + ReportFormat::Latex => self.generate_latex_content(data, config), + ReportFormat::Markdown => self.generate_markdown_content(data, config), + ReportFormat::Csv => self.generate_csv_content(data, config), + } + } +} + +impl PerformanceTableReport { + fn generate_html_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut html = String::from( + r#" + + + Performance Table Report + + + +

    Performance Table Report

    +

    Generated on: "#, + ); + + html.push_str( + &chrono::Utc::now() + .format("%Y-%m-%d %H:%M:%S UTC") + .to_string(), + ); + html.push_str("

    "); + + for (problem, results) in data { + html.push_str(&format!( + r#"

    Problem: {}

    + + + + + + + + + +"#, + problem.get_name() + )); + + let mut optimizer_stats = HashMap::new(); + for result in &results.results { + let stats = optimizer_stats + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + stats.push(result); + } + + let mut perf_data = Vec::new(); + for (optimizer, runs) in &optimizer_stats { + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + + let mean_final = if !final_values.is_empty() { + final_values.iter().sum::() / final_values.len() as f64 + } else { + f64::INFINITY + }; + + let best_final = final_values.iter().cloned().fold(f64::INFINITY, f64::min); + + let mean_func_evals = runs + .iter() + .map(|r| r.function_evaluations as f64) + .sum::() + / runs.len() as f64; + + let mean_time = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .sum::() + / runs.len() as f64; + + perf_data.push(( + optimizer.clone(), + success_rate, + mean_final, + best_final, + mean_func_evals, + mean_time, + )); + } + + // Sort by success rate, then by best value + perf_data.sort_by(|a, b| { + match b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal) { + std::cmp::Ordering::Equal => { + a.3.partial_cmp(&b.3).unwrap_or(std::cmp::Ordering::Equal) + } + other => other, + } + }); + + for ( + i, + (optimizer, success_rate, mean_final, best_final, mean_func_evals, mean_time), + ) in perf_data.iter().enumerate() + { + let class = if i == 0 { " class=\"best\"" } else { "" }; + html.push_str(&format!( + "\n", + class, optimizer, success_rate, mean_final, best_final, mean_func_evals, mean_time + )); + } + + html.push_str("
    OptimizerSuccess Rate (%)Mean Final ValueBest ValueMean Function EvalsMean Time (s)
    {}{:.1}{:.2e}{:.2e}{:.1}{:.3}
    \n"); + } + + html.push_str(""); + Ok(html) + } + + fn generate_latex_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut latex = String::from( + r#"\documentclass{article} +\usepackage[margin=0.5in]{geometry} +\usepackage{booktabs} +\usepackage{array} +\usepackage{siunitx} +\usepackage{longtable} +\title{Performance Table Report} +\author{QQN Optimizer Benchmark} +\date{\today} +\begin{document} +\maketitle + +"#, + ); + + for (problem, results) in data { + latex.push_str(&format!( + r#"\section{{Problem: {}}} +\begin{{longtable}}{{p{{3cm}}*{{5}}{{c}}}} +\toprule +\textbf{{Optimizer}} & \textbf{{Success Rate (\%)}} & \textbf{{Mean Final Value}} & \textbf{{Best Value}} & \textbf{{Mean Func Evals}} & \textbf{{Mean Time (s)}} \\ +\midrule +"#, + self.escape_latex(&problem.get_name()) + )); + + let mut optimizer_stats = HashMap::new(); + for result in &results.results { + let stats = optimizer_stats + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + stats.push(result); + } + + for (optimizer, runs) in &optimizer_stats { + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + + let mean_final = if !final_values.is_empty() { + final_values.iter().sum::() / final_values.len() as f64 + } else { + f64::INFINITY + }; + + let best_final = final_values.iter().cloned().fold(f64::INFINITY, f64::min); + + let mean_func_evals = runs + .iter() + .map(|r| r.function_evaluations as f64) + .sum::() + / runs.len() as f64; + + let mean_time = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .sum::() + / runs.len() as f64; + + latex.push_str(&format!( + "{} & {:.1} & {:.2e} & {:.2e} & {:.1} & {:.3} \\\\\n", + self.escape_latex(optimizer), + success_rate, + mean_final, + best_final, + mean_func_evals, + mean_time + )); + } + + latex.push_str( + r#"\bottomrule +\end{longtable} + +"#, + ); + } + + latex.push_str("\\end{document}"); + Ok(latex) + } + + fn generate_markdown_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut md = String::from("# Performance Table Report\n\n"); + md.push_str(&format!( + "Generated on: {}\n\n", + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC") + )); + + for (problem, results) in data { + md.push_str(&format!("## Problem: {}\n\n", problem.get_name())); + md.push_str("| Optimizer | Success Rate (%) | Mean Final Value | Best Value | Mean Func Evals | Mean Time (s) |\n"); + md.push_str("|-----------|------------------|------------------|------------|-----------------|---------------|\n"); + + let mut optimizer_stats = HashMap::new(); + for result in &results.results { + let stats = optimizer_stats + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + stats.push(result); + } + + for (optimizer, runs) in &optimizer_stats { + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + + let mean_final = if !final_values.is_empty() { + final_values.iter().sum::() / final_values.len() as f64 + } else { + f64::INFINITY + }; + + let best_final = final_values.iter().cloned().fold(f64::INFINITY, f64::min); + + let mean_func_evals = runs + .iter() + .map(|r| r.function_evaluations as f64) + .sum::() + / runs.len() as f64; + + let mean_time = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .sum::() + / runs.len() as f64; + + md.push_str(&format!( + "| {} | {:.1} | {:.2e} | {:.2e} | {:.1} | {:.3} |\n", + optimizer, success_rate, mean_final, best_final, mean_func_evals, mean_time + )); + } + + md.push_str("\n"); + } + + Ok(md) + } + + fn generate_csv_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut csv = String::from("Problem,Optimizer,Success_Rate,Mean_Final_Value,Best_Value,Mean_Func_Evals,Mean_Time\n"); + + for (problem, results) in data { + let problem_name = problem.get_name(); + + let mut optimizer_stats = HashMap::new(); + for result in &results.results { + let stats = optimizer_stats + .entry(result.optimizer_name.clone()) + .or_insert(Vec::new()); + stats.push(result); + } + + for (optimizer, runs) in &optimizer_stats { + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + + let mean_final = if !final_values.is_empty() { + final_values.iter().sum::() / final_values.len() as f64 + } else { + f64::INFINITY + }; + + let best_final = final_values.iter().cloned().fold(f64::INFINITY, f64::min); + + let mean_func_evals = runs + .iter() + .map(|r| r.function_evaluations as f64) + .sum::() + / runs.len() as f64; + + let mean_time = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .sum::() + / runs.len() as f64; + + csv.push_str(&format!( + "{},{},{:.1},{:.2e},{:.2e},{:.1},{:.3}\n", + problem_name, + optimizer, + success_rate, + mean_final, + best_final, + mean_func_evals, + mean_time + )); + } + } + + Ok(csv) + } + + fn escape_latex(&self, text: &str) -> String { + text.replace("_", "\\_") + .replace("&", "\\&") + .replace("%", "\\%") + .replace("$", "\\$") + .replace("#", "\\#") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment_runner::unified_report_tests::UnifiedReportTestSuite; + + #[test] + fn test_performance_table_report_basic() { + let report = PerformanceTableReport::new(); + assert_eq!(report.name(), "performance_table"); + assert!(report.description().contains("performance table")); + } + + #[test] + fn test_performance_table_report_with_unified_suite() { + let report = PerformanceTableReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + + #[test] + fn test_performance_table_all_formats() { + let report = PerformanceTableReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + // Test HTML + let html_config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + let html_content = report.generate_content(&data_refs, &html_config).unwrap(); + assert!(html_content.contains("")); + assert!(html_content.contains("Performance Table Report")); + + // Test Markdown + let md_config = ReportConfig { + format: ReportFormat::Markdown, + ..Default::default() + }; + let md_content = report.generate_content(&data_refs, &md_config).unwrap(); + assert!(md_content.contains("# Performance Table Report")); + assert!(md_content.contains("| Optimizer |")); + + // Test CSV + let csv_config = ReportConfig { + format: ReportFormat::Csv, + ..Default::default() + }; + let csv_content = report.generate_content(&data_refs, &csv_config).unwrap(); + assert!(csv_content.contains("Problem,Optimizer")); + + // Test LaTeX + let latex_config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + let latex_content = report.generate_content(&data_refs, &latex_config).unwrap(); + assert!(latex_content.contains("\\documentclass")); + } +} diff --git a/src/experiment_runner/reports/unified_summary_statistics.rs b/src/experiment_runner/reports/unified_summary_statistics.rs new file mode 100644 index 00000000..44e67fb7 --- /dev/null +++ b/src/experiment_runner/reports/unified_summary_statistics.rs @@ -0,0 +1,461 @@ +//! Summary statistics report implementation using the unified report trait. + +use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; +use crate::experiment_runner::reports::summary_statistics::generate_summary_statistics_table_content; +use crate::experiment_runner::unified_report::{Report, ReportConfig, ReportFormat}; +use anyhow::Result; + +/// Summary statistics report that provides aggregate performance metrics +/// grouped by problem family and optimizer. +pub struct SummaryStatisticsReport; + +impl SummaryStatisticsReport { + /// Create a new summary statistics report + pub fn new() -> Self { + Self + } +} + +impl Default for SummaryStatisticsReport { + fn default() -> Self { + Self::new() + } +} + +impl Report for SummaryStatisticsReport { + fn name(&self) -> &'static str { + "summary_statistics" + } + + fn description(&self) -> &'static str { + "Summary statistics showing average performance metrics grouped by problem family and optimizer" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + self.validate_data(data)?; + + match config.format { + ReportFormat::Html => self.generate_html_content(data, config), + ReportFormat::Latex => self.generate_latex_content(data, config), + ReportFormat::Markdown => self.generate_markdown_content(data, config), + ReportFormat::Csv => self.generate_csv_content(data, config), + } + } +} + +impl SummaryStatisticsReport { + fn generate_html_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut html = String::from( + r#" + + + Summary Statistics Report + + + +

    Summary Statistics Report

    +

    Generated on: "#, + ); + + html.push_str( + &chrono::Utc::now() + .format("%Y-%m-%d %H:%M:%S UTC") + .to_string(), + ); + html.push_str("

    "); + + // Convert LaTeX table to HTML format + let latex_content = generate_summary_statistics_table_content(data)?; + let html_table = self.convert_latex_to_html(&latex_content)?; + html.push_str(&html_table); + + html.push_str(""); + Ok(html) + } + + fn generate_latex_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut latex = String::from( + r#"\documentclass{article} +\usepackage[margin=0.5in]{geometry} +\usepackage{booktabs} +\usepackage{array} +\usepackage{siunitx} +\usepackage{multirow} +\usepackage{colortbl} +\usepackage{xcolor} +\usepackage{adjustbox} +\title{Summary Statistics Report} +\author{QQN Optimizer Benchmark} +\date{\today} +\begin{document} +\maketitle + +"#, + ); + + latex.push_str(&generate_summary_statistics_table_content(data)?); + latex.push_str("\n\\end{document}"); + + Ok(latex) + } + + fn generate_markdown_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + let mut md = String::from("# Summary Statistics Report\n\n"); + md.push_str(&format!( + "Generated on: {}\n\n", + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC") + )); + + // Convert LaTeX table to Markdown format + let latex_content = generate_summary_statistics_table_content(data)?; + let markdown_table = self.convert_latex_to_markdown(&latex_content)?; + md.push_str(&markdown_table); + + Ok(md) + } + + fn generate_csv_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + _config: &ReportConfig, + ) -> Result { + use crate::experiment_runner::report_generator::get_family; + use std::collections::HashMap; + + let mut csv = String::from("Problem_Family,Optimizer,Avg_Success_Rate,Avg_Final_Value,Avg_Func_Evals,Avg_Grad_Evals,Avg_Time\n"); + + // Group by problem family (similar to the original logic) + let mut family_results: HashMap< + String, + HashMap>, + > = HashMap::new(); + for (problem, results) in data { + let family = get_family(&problem.get_name()); + for result in &results.results { + family_results + .entry(family.clone()) + .or_default() + .entry(result.optimizer_name.clone()) + .or_default() + .push(result); + } + } + + let mut families: Vec<_> = family_results.keys().cloned().collect(); + families.sort(); + + for family in families { + if let Some(optimizers) = family_results.get(&family) { + for (optimizer, runs) in optimizers { + let success_count = runs.iter().filter(|r| r.convergence_achieved).count(); + let success_rate = success_count as f64 / runs.len() as f64 * 100.0; + + let final_values: Vec = runs + .iter() + .map(|r| r.final_value) + .filter(|&v| v.is_finite()) + .collect(); + let avg_final = if !final_values.is_empty() { + final_values.iter().sum::() / final_values.len() as f64 + } else { + f64::INFINITY + }; + + let avg_func_evals = runs + .iter() + .map(|r| r.function_evaluations as f64) + .sum::() + / runs.len() as f64; + let avg_grad_evals = runs + .iter() + .map(|r| r.gradient_evaluations as f64) + .sum::() + / runs.len() as f64; + let avg_time = runs + .iter() + .map(|r| r.execution_time.as_secs_f64()) + .sum::() + / runs.len() as f64; + + csv.push_str(&format!( + "{},{},{:.1},{:.2e},{:.1},{:.1},{:.3}\n", + family, + optimizer, + success_rate, + avg_final, + avg_func_evals, + avg_grad_evals, + avg_time + )); + } + } + } + + Ok(csv) + } + + fn convert_latex_to_html(&self, latex_content: &str) -> Result { + // Simple LaTeX to HTML conversion for tables + // This is a basic implementation - a full converter would be more complex + let mut html = String::new(); + + // Extract table content between \begin{tabular} and \end{tabular} + if let Some(start) = latex_content.find("\\begin{tabular}") { + if let Some(end) = latex_content.find("\\end{tabular}") { + html.push_str("\n"); + + let table_content = &latex_content[start..end]; + let lines: Vec<&str> = table_content.lines().collect(); + + let mut in_header = true; + for line in lines { + let trimmed = line.trim(); + if trimmed.contains("\\toprule") + || trimmed.contains("\\midrule") + || trimmed.contains("\\bottomrule") + { + continue; + } + if trimmed.contains("&") && trimmed.ends_with("\\\\") { + let row_data = trimmed.trim_end_matches("\\\\"); + let cells: Vec<&str> = row_data.split(" & ").collect(); + + if in_header { + html.push_str(" \n"); + for cell in cells { + let clean_cell = cell.replace("\\textbf{", "").replace("}", ""); + html.push_str(&format!(" \n", clean_cell)); + } + html.push_str(" \n"); + in_header = false; + } else { + html.push_str(" \n"); + for cell in cells { + let clean_cell = cell.replace("\\textbf{", "").replace("}", ""); + html.push_str(&format!(" \n", clean_cell)); + } + html.push_str(" \n"); + } + } + } + + html.push_str("
    {}
    {}
    \n"); + } + } + + if html.is_empty() { + html = "

    Table content could not be converted

    ".to_string(); + } + + Ok(html) + } + + fn convert_latex_to_markdown(&self, latex_content: &str) -> Result { + // Simple LaTeX to Markdown conversion for tables + let mut md = String::new(); + + if let Some(start) = latex_content.find("\\begin{tabular}") { + if let Some(end) = latex_content.find("\\end{tabular}") { + let table_content = &latex_content[start..end]; + let lines: Vec<&str> = table_content.lines().collect(); + + let mut header_written = false; + for line in lines { + let trimmed = line.trim(); + if trimmed.contains("\\toprule") + || trimmed.contains("\\midrule") + || trimmed.contains("\\bottomrule") + { + continue; + } + if trimmed.contains("&") && trimmed.ends_with("\\\\") { + let row_data = trimmed.trim_end_matches("\\\\"); + let cells: Vec<&str> = row_data.split(" & ").collect(); + + if !header_written { + // Write header + md.push('|'); + for cell in &cells { + let clean_cell = cell.replace("\\textbf{", "").replace("}", ""); + md.push_str(&format!(" {} |", clean_cell)); + } + md.push('\n'); + + // Write separator + md.push('|'); + for _ in &cells { + md.push_str(" --- |"); + } + md.push('\n'); + header_written = true; + } else { + // Write data row + md.push('|'); + for cell in cells { + let clean_cell = cell.replace("\\textbf{", "").replace("}", ""); + md.push_str(&format!(" {} |", clean_cell)); + } + md.push('\n'); + } + } + } + } + } + + if md.is_empty() { + md = "Table content could not be converted\n".to_string(); + } + + Ok(md) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::benchmarks::evaluation::{ + BenchmarkConfig, ConvergenceReason, OptimizationTrace, PerformanceMetrics, SingleResult, + }; + use crate::SphereFunction; + use std::sync::Arc; + use std::time::Duration; + + fn create_test_data() -> Vec<(ProblemSpec, BenchmarkResults)> { + let problem_spec = ProblemSpec { + name: None, + problem: Arc::new(SphereFunction::new(2)), + dimensions: Some(2), + family: "Convex Unimodal".to_string(), + seed: 0, + }; + + let result = SingleResult { + problem_name: "".to_string(), + optimizer_name: "TestOptimizer".to_string(), + run_id: 0, + final_value: 1e-6, + final_gradient_norm: 1e-8, + iterations: 100, + function_evaluations: 150, + gradient_evaluations: 100, + execution_time: Duration::from_millis(100), + convergence_achieved: true, + convergence_reason: ConvergenceReason::FunctionTolerance, + memory_usage: None, + best_value: 1e-6, + trace: OptimizationTrace::default(), + error_message: None, + performance_metrics: PerformanceMetrics { + iterations_per_second: 0.0, + function_evaluations_per_second: 0.0, + gradient_evaluations_per_second: 0.0, + convergence_rate: 0.0, + }, + }; + + let results = BenchmarkResults { + config: BenchmarkConfig::default(), + timestamp: Default::default(), + convergence_achieved: false, + final_value: None, + function_evaluations: 0, + results: vec![result], + gradient_evaluations: 0, + }; + + vec![(problem_spec, results)] + } + + #[test] + fn test_summary_statistics_report_basic() { + let report = SummaryStatisticsReport::new(); + assert_eq!(report.name(), "summary_statistics"); + assert!(report.description().contains("Summary statistics")); + } + + #[test] + fn test_summary_statistics_report_html() { + let report = SummaryStatisticsReport::new(); + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!(content.contains("")); + assert!(content.contains("Summary Statistics Report")); + } + + #[test] + fn test_summary_statistics_report_latex() { + let report = SummaryStatisticsReport::new(); + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!(content.contains("\\documentclass")); + assert!(content.contains("Summary Statistics Report")); + } + + #[test] + fn test_summary_statistics_report_markdown() { + let report = SummaryStatisticsReport::new(); + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Markdown, + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!(content.contains("# Summary Statistics Report")); + } + + #[test] + fn test_summary_statistics_report_csv() { + let report = SummaryStatisticsReport::new(); + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Csv, + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!(content.contains("Problem_Family,Optimizer")); + assert!(content.contains("Convex Unimodal,TestOptimizer")); + } +} diff --git a/src/experiment_runner/unified_report.rs b/src/experiment_runner/unified_report.rs new file mode 100644 index 00000000..590cd016 --- /dev/null +++ b/src/experiment_runner/unified_report.rs @@ -0,0 +1,409 @@ +//! Unified trait for all report types. +//! +//! This module provides a common interface for all report generation, +//! enabling consistent testing and usage patterns across different report types. + +use crate::benchmarks::evaluation::{BenchmarkResults, ProblemSpec}; +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; + +/// Configuration for report generation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReportConfig { + /// Output format (html, latex, csv, markdown) + pub format: ReportFormat, + /// Include detailed statistics + pub include_detailed_stats: bool, + /// Include plots and visualizations + pub include_plots: bool, + /// Custom styling options + pub style_options: HashMap, +} + +/// Supported report output formats +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)] +pub enum ReportFormat { + Html, + Latex, + Csv, + Markdown, +} + +impl Default for ReportConfig { + fn default() -> Self { + Self { + format: ReportFormat::Html, + include_detailed_stats: true, + include_plots: true, + style_options: HashMap::new(), + } + } +} + +/// Metadata about a generated report +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReportMetadata { + /// Report type identifier + pub report_type: String, + /// Generation timestamp + pub generated_at: chrono::DateTime, + /// Number of problems analyzed + pub problem_count: usize, + /// Number of optimizers compared + pub optimizer_count: usize, + /// Total data points processed + pub data_points: usize, +} + +/// Unified trait for all report types +pub trait Report { + /// Get the name/identifier for this report type + fn name(&self) -> &'static str; + + /// Get a description of what this report provides + fn description(&self) -> &'static str; + + /// Generate the report content for the specified format + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result; + + /// Export the report to a file + fn export_to_file( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_path: &Path, + ) -> Result<()> { + let content = self.generate_content(data, config)?; + std::fs::write(output_path, content)?; + Ok(()) + } + + /// Validate that the input data is suitable for this report + fn validate_data(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> Result<()> { + if data.is_empty() { + anyhow::bail!("Cannot generate {} report: no data provided", self.name()); + } + + for (problem, results) in data { + if results.results.is_empty() { + anyhow::bail!( + "Cannot generate {} report: no results for problem '{}'", + self.name(), + problem.get_name() + ); + } + } + + Ok(()) + } + + /// Get metadata about the generated report + fn get_metadata(&self, data: &[(&ProblemSpec, BenchmarkResults)]) -> ReportMetadata { + let problem_count = data.len(); + let optimizer_count = data + .iter() + .flat_map(|(_, results)| &results.results) + .map(|r| &r.optimizer_name) + .collect::>() + .len(); + let data_points = data.iter().map(|(_, results)| results.results.len()).sum(); + + ReportMetadata { + report_type: self.name().to_string(), + generated_at: chrono::Utc::now(), + problem_count, + optimizer_count, + data_points, + } + } + + /// Get supported output formats for this report type + fn supported_formats(&self) -> Vec { + vec![ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ] + } +} + +/// Collection of reports for batch processing +#[derive(Default)] +pub struct ReportCollection { + reports: Vec>, +} + +impl ReportCollection { + /// Create a new empty report collection + pub fn new() -> Self { + Self::default() + } + + /// Add a report to the collection + pub fn add_report(mut self, report: R) -> Self { + self.reports.push(Box::new(report)); + self + } + + /// Generate all reports in the collection + pub fn generate_all( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + output_dir: &Path, + ) -> Result> { + let mut metadata = Vec::new(); + + std::fs::create_dir_all(output_dir)?; + + for report in &self.reports { + report.validate_data(data)?; + + let filename = format!( + "{}.{}", + report.name(), + match config.format { + ReportFormat::Html => "html", + ReportFormat::Latex => "tex", + ReportFormat::Csv => "csv", + ReportFormat::Markdown => "md", + } + ); + + let output_path = output_dir.join(filename); + report.export_to_file(data, config, &output_path)?; + + metadata.push(report.get_metadata(data)); + } + + Ok(metadata) + } + + /// Get all report names in the collection + pub fn report_names(&self) -> Vec<&'static str> { + self.reports.iter().map(|r| r.name()).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::benchmarks::evaluation::{ + BenchmarkConfig, BenchmarkResults, ConvergenceReason, OptimizationTrace, + PerformanceMetrics, ProblemSpec, SingleResult, + }; + use crate::experiment_runner::{Report, ReportCollection, ReportFormat}; + use crate::SphereFunction; + use std::sync::Arc; + use std::time::Duration; + + // Mock report implementation for testing + struct MockReport { + name: &'static str, + } + + impl Report for MockReport { + fn name(&self) -> &'static str { + self.name + } + + fn description(&self) -> &'static str { + "Mock report for testing" + } + + fn generate_content( + &self, + data: &[(&ProblemSpec, BenchmarkResults)], + config: &ReportConfig, + ) -> Result { + self.validate_data(data)?; + + match config.format { + ReportFormat::Html => Ok(format!( + "

    {}

    Problems: {}

    ", + self.name(), + data.len() + )), + ReportFormat::Markdown => { + Ok(format!("# {}\n\nProblems: {}\n", self.name(), data.len())) + } + ReportFormat::Latex => Ok(format!( + "\\section{{{}}}\nProblems: {}\n", + self.name(), + data.len() + )), + ReportFormat::Csv => Ok(format!( + "report_type,problem_count\n{},{}\n", + self.name(), + data.len() + )), + } + } + } + + fn create_test_data() -> Vec<(ProblemSpec, BenchmarkResults)> { + // Create minimal test data + let problem_spec = ProblemSpec { + name: None, + problem: Arc::new(SphereFunction::new(2)), + dimensions: Some(2), + family: "Test".to_string(), + seed: 0, + }; + + let result = SingleResult { + problem_name: "".to_string(), + optimizer_name: "TestOptimizer".to_string(), + run_id: 0, + final_value: 1e-6, + final_gradient_norm: 1e-8, + iterations: 100, + function_evaluations: 150, + gradient_evaluations: 100, + execution_time: Duration::from_millis(100), + convergence_achieved: true, + convergence_reason: ConvergenceReason::FunctionTolerance, + memory_usage: None, + best_value: 1e-6, + trace: OptimizationTrace::default(), + error_message: None, + performance_metrics: PerformanceMetrics { + iterations_per_second: 0.0, + function_evaluations_per_second: 0.0, + gradient_evaluations_per_second: 0.0, + convergence_rate: 0.0, + }, + }; + + let results = BenchmarkResults { + config: BenchmarkConfig::default(), + timestamp: Default::default(), + convergence_achieved: false, + final_value: None, + function_evaluations: 0, + results: vec![result], + gradient_evaluations: 0, + }; + + vec![(problem_spec, results)] + } + + #[test] + fn test_report_trait_basic_functionality() { + let report = MockReport { + name: "test_report", + }; + assert_eq!(report.name(), "test_report"); + assert_eq!(report.description(), "Mock report for testing"); + + let formats = report.supported_formats(); + assert!(formats.contains(&ReportFormat::Html)); + assert!(formats.contains(&ReportFormat::Latex)); + assert!(formats.contains(&ReportFormat::Markdown)); + } + + #[test] + fn test_report_content_generation() { + let report = MockReport { + name: "test_report", + }; + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + assert!(content.contains("

    test_report

    ")); + assert!(content.contains("Problems: 1")); + } + + #[test] + fn test_report_validation() { + let report = MockReport { + name: "test_report", + }; + + // Test empty data validation + let empty_data = vec![]; + assert!(report.validate_data(&empty_data).is_err()); + + // Test valid data validation + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + assert!(report.validate_data(&data_refs).is_ok()); + } + + #[test] + fn test_report_metadata() { + let report = MockReport { + name: "test_report", + }; + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let metadata = report.get_metadata(&data_refs); + assert_eq!(metadata.report_type, "test_report"); + assert_eq!(metadata.problem_count, 1); + assert_eq!(metadata.optimizer_count, 1); + assert_eq!(metadata.data_points, 1); + } + + #[test] + fn test_report_collection() { + let mut collection = ReportCollection::new(); + collection = collection + .add_report(MockReport { name: "report1" }) + .add_report(MockReport { name: "report2" }); + + let names = collection.report_names(); + assert_eq!(names.len(), 2); + assert!(names.contains(&"report1")); + assert!(names.contains(&"report2")); + } + + #[test] + fn test_different_output_formats() { + let report = MockReport { + name: "test_report", + }; + let data = create_test_data(); + let data_refs: Vec<_> = data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let html_config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + let html_content = report.generate_content(&data_refs, &html_config).unwrap(); + assert!(html_content.contains("

    ")); + + let md_config = ReportConfig { + format: ReportFormat::Markdown, + ..Default::default() + }; + let md_content = report.generate_content(&data_refs, &md_config).unwrap(); + assert!(md_content.contains("# test_report")); + + let latex_config = ReportConfig { + format: ReportFormat::Latex, + ..Default::default() + }; + let latex_content = report.generate_content(&data_refs, &latex_config).unwrap(); + assert!(latex_content.contains("\\section{")); + + let csv_config = ReportConfig { + format: ReportFormat::Csv, + ..Default::default() + }; + let csv_content = report.generate_content(&data_refs, &csv_config).unwrap(); + assert!(csv_content.contains("report_type,problem_count")); + } +} diff --git a/src/experiment_runner/unified_report_example.rs b/src/experiment_runner/unified_report_example.rs new file mode 100644 index 00000000..64d76cfc --- /dev/null +++ b/src/experiment_runner/unified_report_example.rs @@ -0,0 +1,161 @@ +//! Example demonstrating how to use the unified reporting system. +//! +//! This example shows how to create reports using the unified trait, +//! configure them, and generate outputs in multiple formats. + +use crate::experiment_runner::reports::unified_performance_table::PerformanceTableReport; +use crate::experiment_runner::reports::unified_summary_statistics::SummaryStatisticsReport; +use crate::experiment_runner::{ + Report, ReportCollection, ReportConfig, ReportFormat, UnifiedReportTestSuite, +}; +use anyhow::Result; +use std::path::Path; +use tempfile::TempDir; + +/// Example usage of the unified reporting system +pub struct UnifiedReportingExample; + +impl UnifiedReportingExample { + /// Generate all reports in multiple formats + pub fn generate_comprehensive_reports(output_dir: &Path) -> Result<()> { + // Create test data (in practice, this would come from actual benchmark results) + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + // Create report collection with multiple report types + let reports = ReportCollection::new() + .add_report(SummaryStatisticsReport::new()) + .add_report(PerformanceTableReport::new()); + + // Generate reports in different formats + let formats = vec![ + ReportFormat::Html, + ReportFormat::Markdown, + ReportFormat::Latex, + ReportFormat::Csv, + ]; + + for format in formats { + let format_dir = output_dir.join(format!("{:?}", format).to_lowercase()); + std::fs::create_dir_all(&format_dir)?; + + let config = ReportConfig { + format, + include_detailed_stats: true, + include_plots: false, // We don't have plotting in this example + ..Default::default() + }; + + let metadata_list = reports.generate_all(&data_refs, &config, &format_dir)?; + println!( + "Generated {} reports in {:?} format", + metadata_list.len(), + config.format + ); + for metadata in metadata_list { + println!(" - {} ({})", metadata.report_type, metadata.data_points); + } + } + + Ok(()) + } + + /// Example of using a single report with custom configuration + pub fn generate_custom_report() -> Result { + let report = SummaryStatisticsReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let mut style_options = std::collections::HashMap::new(); + style_options.insert("theme".to_string(), "dark".to_string()); + style_options.insert("font_size".to_string(), "12px".to_string()); + + let config = ReportConfig { + format: ReportFormat::Html, + include_detailed_stats: true, + include_plots: false, + style_options, + }; + + let content = report.generate_content(&data_refs, &config)?; + + // Validate the report + report.validate_data(&data_refs)?; + + // Get metadata + let metadata = report.get_metadata(&data_refs); + println!( + "Generated {} report with {} data points", + metadata.report_type, metadata.data_points + ); + + Ok(content) + } + + /// Example of validating reports using the unified test suite + pub fn validate_all_reports() -> Result<()> { + let reports: Vec> = vec![ + Box::new(SummaryStatisticsReport::new()), + Box::new(PerformanceTableReport::new()), + ]; + + for report in reports { + println!("Validating {} report...", report.name()); + + // Run comprehensive tests + UnifiedReportTestSuite::test_basic_functionality(report.as_ref())?; + UnifiedReportTestSuite::test_content_generation(report.as_ref())?; + UnifiedReportTestSuite::test_data_validation(report.as_ref())?; + UnifiedReportTestSuite::test_metadata_generation(report.as_ref())?; + UnifiedReportTestSuite::test_file_export(report.as_ref())?; + UnifiedReportTestSuite::test_all_formats(report.as_ref())?; + + println!("✓ {} report validation passed", report.name()); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_comprehensive_report_generation() { + let temp_dir = TempDir::new().unwrap(); + UnifiedReportingExample::generate_comprehensive_reports(temp_dir.path()).unwrap(); + + // Verify output directories and files were created + for format in &["html", "markdown", "latex", "csv"] { + let format_dir = temp_dir.path().join(format); + assert!( + format_dir.exists(), + "Format directory should exist: {}", + format + ); + + // Check that report files were created + let entries: Vec<_> = std::fs::read_dir(&format_dir).unwrap().collect(); + assert!( + !entries.is_empty(), + "Should have generated report files in {}", + format + ); + } + } + + #[test] + fn test_custom_report_generation() { + let content = UnifiedReportingExample::generate_custom_report().unwrap(); + assert!(!content.is_empty()); + assert!(content.contains("")); + assert!(content.contains("Summary Statistics Report")); + } + + #[test] + fn test_report_validation() { + // This should not panic or return errors + UnifiedReportingExample::validate_all_reports().unwrap(); + } +} diff --git a/src/experiment_runner/unified_report_tests.rs b/src/experiment_runner/unified_report_tests.rs new file mode 100644 index 00000000..759c1bdb --- /dev/null +++ b/src/experiment_runner/unified_report_tests.rs @@ -0,0 +1,442 @@ +//! Comprehensive unified testing infrastructure for all report types. +//! +//! This module provides unified testing patterns that can be applied to any +//! report implementation using the unified Report trait. + +use crate::benchmarks::evaluation::IterationData; +use crate::benchmarks::evaluation::{ + BenchmarkConfig, BenchmarkResults, ConvergenceReason, DurationWrapper, OptimizationTrace, + PerformanceMetrics, ProblemSpec, SingleResult, +}; +use crate::experiment_runner::reports::family_vs_family_report::FamilyVsFamilyReport; +use crate::experiment_runner::reports::unified_summary_statistics::SummaryStatisticsReport; +use crate::experiment_runner::unified_report::{ + Report, ReportCollection, ReportConfig, ReportFormat, +}; +use crate::SphereFunction; +use anyhow::Result; +use std::sync::Arc; +use std::time::Duration; +use tempfile::TempDir; + +/// Test suite that validates any Report implementation +pub struct UnifiedReportTestSuite; + +impl UnifiedReportTestSuite { + /// Run all standard tests on a report implementation + pub fn test_report(report: &R) -> Result<()> { + Self::test_basic_functionality(report)?; + Self::test_content_generation(report)?; + Self::test_data_validation(report)?; + Self::test_metadata_generation(report)?; + Self::test_file_export(report)?; + Self::test_all_formats(report)?; + Ok(()) + } + + /// Test basic functionality of a report + pub fn test_basic_functionality(report: &R) -> Result<()> { + // Test name is not empty + assert!(!report.name().is_empty(), "Report name should not be empty"); + + // Test description is not empty + assert!( + !report.description().is_empty(), + "Report description should not be empty" + ); + + // Test supported formats + let formats = report.supported_formats(); + assert!( + !formats.is_empty(), + "Report should support at least one format" + ); + + println!("✓ Basic functionality test passed for {}", report.name()); + Ok(()) + } + + /// Test content generation for different formats + pub fn test_content_generation(report: &R) -> Result<()> { + let test_data = Self::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + for format in &report.supported_formats() { + let config = ReportConfig { + format: format.clone(), + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config)?; + assert!( + !content.is_empty(), + "Generated content should not be empty for format {:?}", + format + ); + + // Format-specific validations + match format { + ReportFormat::Html => { + assert!( + content.contains("<") && content.contains(">"), + "HTML content should contain HTML tags" + ); + } + ReportFormat::Latex => { + assert!( + content.contains("\\"), + "LaTeX content should contain LaTeX commands" + ); + } + ReportFormat::Markdown => { + assert!( + content.contains("#") || content.contains("|"), + "Markdown content should contain markdown syntax" + ); + } + ReportFormat::Csv => { + assert!( + content.contains(","), + "CSV content should contain comma separators" + ); + } + } + } + + println!("✓ Content generation test passed for {}", report.name()); + Ok(()) + } + + /// Test data validation + pub fn test_data_validation(report: &R) -> Result<()> { + // Test empty data validation + let empty_data = vec![]; + assert!( + report.validate_data(&empty_data).is_err(), + "Should reject empty data" + ); + + // Test valid data validation + let test_data = Self::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + assert!( + report.validate_data(&data_refs).is_ok(), + "Should accept valid data" + ); + + // Test data with empty results + let empty_results_data = Self::create_empty_results_data(); + let empty_refs: Vec<_> = empty_results_data + .iter() + .map(|(p, r)| (p, r.clone())) + .collect(); + assert!( + report.validate_data(&empty_refs).is_err(), + "Should reject data with empty results" + ); + + println!("✓ Data validation test passed for {}", report.name()); + Ok(()) + } + + /// Test metadata generation + pub fn test_metadata_generation(report: &R) -> Result<()> { + let test_data = Self::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let metadata = report.get_metadata(&data_refs); + + assert_eq!(metadata.report_type, report.name()); + assert!( + metadata.problem_count > 0, + "Should have non-zero problem count" + ); + assert!( + metadata.optimizer_count > 0, + "Should have non-zero optimizer count" + ); + assert!(metadata.data_points > 0, "Should have non-zero data points"); + + println!("✓ Metadata generation test passed for {}", report.name()); + Ok(()) + } + + /// Test file export functionality + pub fn test_file_export(report: &R) -> Result<()> { + let test_data = Self::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let temp_dir = TempDir::new()?; + + for format in &report.supported_formats() { + let config = ReportConfig { + format: format.clone(), + ..Default::default() + }; + + let filename = format!( + "{}.{}", + report.name(), + match format { + ReportFormat::Html => "html", + ReportFormat::Latex => "tex", + ReportFormat::Csv => "csv", + ReportFormat::Markdown => "md", + } + ); + + let output_path = temp_dir.path().join(filename); + report.export_to_file(&data_refs, &config, &output_path)?; + + assert!(output_path.exists(), "Output file should be created"); + let file_content = std::fs::read_to_string(&output_path)?; + assert!(!file_content.is_empty(), "Output file should not be empty"); + } + + println!("✓ File export test passed for {}", report.name()); + Ok(()) + } + + /// Test all supported formats work + pub fn test_all_formats(report: &R) -> Result<()> { + let test_data = Self::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + for format in &[ + ReportFormat::Html, + ReportFormat::Latex, + ReportFormat::Markdown, + ReportFormat::Csv, + ] { + if report.supported_formats().contains(format) { + let config = ReportConfig { + format: format.clone(), + ..Default::default() + }; + + let result = report.generate_content(&data_refs, &config); + assert!(result.is_ok(), "Format {:?} should be supported", format); + } + } + + println!("✓ All formats test passed for {}", report.name()); + Ok(()) + } + + /// Create test data for reports + pub fn create_test_data() -> Vec<(ProblemSpec, BenchmarkResults)> { + let mut test_data = Vec::new(); + + // Create test data for multiple problems and optimizers + let problems = vec![ + ("Sphere", "Convex Unimodal"), + ("Rosenbrock", "Non-Convex Unimodal"), + ("Rastrigin", "Highly Multimodal"), + ]; + + let optimizers = vec!["QQN-Basic", "L-BFGS", "Adam", "GD"]; + + for (prob_name, family) in problems { + let problem_spec = ProblemSpec { + name: Some(prob_name.to_string()), + problem: Arc::new(SphereFunction::new(2)), + dimensions: Some(2), + family: family.to_string(), + seed: 0, + }; + + let mut results = Vec::new(); + + for (i, optimizer) in optimizers.iter().enumerate() { + for run_id in 0..3 { + // 3 runs per optimizer + // Create trace data with iterations for convergence analysis + let mut trace = OptimizationTrace::default(); + if run_id < 2 { + // Only successful runs have trace data + let num_iterations = 100 + i * 20; + for iter in 0..num_iterations { + let progress = iter as f64 / num_iterations as f64; + let function_value = 1e-6 * (i + 1) as f64 * (1.0 - progress * 0.9); + trace.iterations.push(IterationData { + iteration: iter, + function_value, + gradient_norm: 1e-8 * (1.0 - progress * 0.9), + step_size: 0.01, + parameters: vec![], + timestamp: DurationWrapper::from(Duration::from_millis( + iter as u64, + )), + total_function_evaluations: 0, + total_gradient_evaluations: 0, + }); + } + } + + let result = SingleResult { + problem_name: "".to_string(), + optimizer_name: optimizer.to_string(), + run_id, + final_value: 1e-6 * (i + 1) as f64, // Different performance + final_gradient_norm: 1e-8, + iterations: 100 + i * 20, + function_evaluations: 150 + i * 30, + gradient_evaluations: 100 + i * 20, + execution_time: Duration::from_millis(100 + i as u64 * 50), + convergence_achieved: run_id < 2, // 2 out of 3 runs succeed + convergence_reason: if run_id < 2 { + ConvergenceReason::FunctionTolerance + } else { + ConvergenceReason::MaxIterations + }, + memory_usage: None, + best_value: 1e-6 * (i + 1) as f64, + trace, + error_message: None, + performance_metrics: PerformanceMetrics::default(), + }; + results.push(result); + } + } + + let benchmark_results = BenchmarkResults { + config: BenchmarkConfig::default(), + timestamp: Default::default(), + convergence_achieved: false, + final_value: None, + function_evaluations: 0, + results, + gradient_evaluations: 0, + }; + + test_data.push((problem_spec, benchmark_results)); + } + + test_data + } + + /// Create test data with empty results (for validation testing) + pub fn create_empty_results_data() -> Vec<(ProblemSpec, BenchmarkResults)> { + let problem_spec = ProblemSpec { + name: None, + problem: Arc::new(SphereFunction::new(2)), + dimensions: Some(2), + family: "Test".to_string(), + seed: 0, + }; + + let benchmark_results = BenchmarkResults { + config: BenchmarkConfig::default(), + timestamp: Default::default(), + convergence_achieved: false, + final_value: None, + function_evaluations: 0, + results: vec![], // Empty results + gradient_evaluations: 0, + }; + + vec![(problem_spec, benchmark_results)] + } +} + +/// Integration tests for the unified reporting system +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_summary_statistics_report_with_unified_suite() { + let report = SummaryStatisticsReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + #[test] + fn test_family_vs_family_report_with_unified_suite() { + let report = FamilyVsFamilyReport::new(); + UnifiedReportTestSuite::test_report(&report).unwrap(); + } + + #[test] + fn test_report_collection() { + let collection = ReportCollection::new() + .add_report(SummaryStatisticsReport::new()) + .add_report(FamilyVsFamilyReport::new()); + + let mut names = collection.report_names(); + names.sort(); + assert_eq!(names, vec!["family_vs_family", "summary_statistics"]); + + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let temp_dir = TempDir::new().unwrap(); + let config = ReportConfig::default(); + + let metadata_list = collection + .generate_all(&data_refs, &config, temp_dir.path()) + .unwrap(); + assert_eq!(metadata_list.len(), 2); + + let report_types: Vec<_> = metadata_list + .iter() + .map(|m| m.report_type.as_str()) + .collect(); + assert!(report_types.contains(&"summary_statistics")); + assert!(report_types.contains(&"family_vs_family")); + } + + #[test] + fn test_unified_test_suite_create_test_data() { + let test_data = UnifiedReportTestSuite::create_test_data(); + assert!(!test_data.is_empty()); + + // Verify structure + for (problem_spec, benchmark_results) in &test_data { + assert!(!problem_spec.get_name().is_empty()); + assert!(!benchmark_results.results.is_empty()); + } + } + + #[test] + fn test_unified_test_suite_validation() { + let report = SummaryStatisticsReport::new(); + + // Test individual validation methods + UnifiedReportTestSuite::test_basic_functionality(&report).unwrap(); + UnifiedReportTestSuite::test_content_generation(&report).unwrap(); + UnifiedReportTestSuite::test_data_validation(&report).unwrap(); + UnifiedReportTestSuite::test_metadata_generation(&report).unwrap(); + UnifiedReportTestSuite::test_file_export(&report).unwrap(); + UnifiedReportTestSuite::test_all_formats(&report).unwrap(); + } + + #[test] + fn test_report_formats_consistency() { + let report = SummaryStatisticsReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + // Test that all supported formats generate different but valid content + let mut contents = std::collections::HashMap::new(); + + for format in &report.supported_formats() { + let config = ReportConfig { + format: format.clone(), + ..Default::default() + }; + + let content = report.generate_content(&data_refs, &config).unwrap(); + contents.insert(format.clone(), content); + } + + // All formats should produce different content + let values: Vec<_> = contents.values().collect(); + for i in 0..values.len() { + for j in i + 1..values.len() { + assert_ne!( + values[i], values[j], + "Different formats should produce different content" + ); + } + } + } +} diff --git a/tests/unified_reports.rs b/tests/unified_reports.rs new file mode 100644 index 00000000..9543542c --- /dev/null +++ b/tests/unified_reports.rs @@ -0,0 +1,150 @@ +// Integration test for unified reporting functionality +use qqn_optimizer::experiment_runner::reports::unified_performance_table::PerformanceTableReport; +use qqn_optimizer::experiment_runner::reports::unified_summary_statistics::SummaryStatisticsReport; +use qqn_optimizer::experiment_runner::{ + Report, ReportCollection, ReportConfig, ReportFormat, UnifiedReportTestSuite, +}; + +#[test] +fn test_unified_report_trait() { + let summary_report = SummaryStatisticsReport::new(); + + // Test basic trait functionality + assert_eq!(summary_report.name(), "summary_statistics"); + assert!(!summary_report.description().is_empty()); + + let formats = summary_report.supported_formats(); + assert!(!formats.is_empty()); + assert!(formats.contains(&ReportFormat::Html)); + assert!(formats.contains(&ReportFormat::Latex)); + assert!(formats.contains(&ReportFormat::Markdown)); + + let performance_report = PerformanceTableReport::new(); + assert_eq!(performance_report.name(), "performance_table"); + assert!(!performance_report.description().is_empty()); +} + +#[test] +fn test_unified_report_collection() { + let collection = ReportCollection::new() + .add_report(SummaryStatisticsReport::new()) + .add_report(PerformanceTableReport::new()); + + let names = collection.report_names(); + assert_eq!(names.len(), 2); + assert!(names.contains(&"summary_statistics")); + assert!(names.contains(&"performance_table")); +} + +#[test] +fn test_unified_test_suite() { + let summary_report = SummaryStatisticsReport::new(); + let performance_report = PerformanceTableReport::new(); + + // Use the unified test suite to validate both reports + UnifiedReportTestSuite::test_basic_functionality(&summary_report).unwrap(); + UnifiedReportTestSuite::test_basic_functionality(&performance_report).unwrap(); + + // Create test data + let test_data = UnifiedReportTestSuite::create_test_data(); + assert!(!test_data.is_empty()); + + for (problem_spec, benchmark_results) in &test_data { + assert!(!problem_spec.get_name().is_empty()); + assert!(!benchmark_results.results.is_empty()); + } +} + +#[test] +fn test_summary_statistics_report_content() { + let report = SummaryStatisticsReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + // Test HTML output + let html_config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + let html_content = report.generate_content(&data_refs, &html_config).unwrap(); + assert!(html_content.contains("")); + assert!(html_content.contains("Summary Statistics Report")); + + // Test Markdown output + let md_config = ReportConfig { + format: ReportFormat::Markdown, + ..Default::default() + }; + let md_content = report.generate_content(&data_refs, &md_config).unwrap(); + assert!(md_content.contains("# Summary Statistics Report")); + + // Test CSV output + let csv_config = ReportConfig { + format: ReportFormat::Csv, + ..Default::default() + }; + let csv_content = report.generate_content(&data_refs, &csv_config).unwrap(); + assert!(csv_content.contains("Problem_Family,Optimizer")); + assert!(csv_content.contains("Convex Unimodal")); +} + +#[test] +fn test_performance_table_report_content() { + let report = PerformanceTableReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + // Test HTML output + let html_config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + let html_content = report.generate_content(&data_refs, &html_config).unwrap(); + assert!(html_content.contains("")); + assert!(html_content.contains("Performance Table Report")); + assert!(html_content.contains("")); + + // Test Markdown output + let md_config = ReportConfig { + format: ReportFormat::Markdown, + ..Default::default() + }; + let md_content = report.generate_content(&data_refs, &md_config).unwrap(); + assert!(md_content.contains("# Performance Table Report")); + assert!(md_content.contains("| Optimizer |")); + + // Test CSV output + let csv_config = ReportConfig { + format: ReportFormat::Csv, + ..Default::default() + }; + let csv_content = report.generate_content(&data_refs, &csv_config).unwrap(); + assert!(csv_content.contains("Problem,Optimizer")); +} + +#[test] +fn test_multiple_reports_different_content() { + let summary_report = SummaryStatisticsReport::new(); + let performance_report = PerformanceTableReport::new(); + let test_data = UnifiedReportTestSuite::create_test_data(); + let data_refs: Vec<_> = test_data.iter().map(|(p, r)| (p, r.clone())).collect(); + + let config = ReportConfig { + format: ReportFormat::Html, + ..Default::default() + }; + + let summary_content = summary_report + .generate_content(&data_refs, &config) + .unwrap(); + let performance_content = performance_report + .generate_content(&data_refs, &config) + .unwrap(); + + // Different reports should produce different content + assert_ne!(summary_content, performance_content); + + // But both should be valid HTML + assert!(summary_content.contains("")); + assert!(performance_content.contains("")); +}