Skip to content

Commit 965f6a0

Browse files
acharneskiacharneski
committed
wip
1 parent 3db361f commit 965f6a0

File tree

9 files changed

+887
-322
lines changed

9 files changed

+887
-322
lines changed

‎docs/result_analysis.md‎

Lines changed: 648 additions & 114 deletions
Large diffs are not rendered by default.

‎paper/combined.md‎

Lines changed: 85 additions & 78 deletions
Large diffs are not rendered by default.

‎paper/content.tex‎

Lines changed: 97 additions & 108 deletions
Large diffs are not rendered by default.

‎paper/paper.pdf‎

-225 KB
Binary file not shown.

‎paper/paper.tex‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,16 @@
1515
\usepackage{multirow}
1616
\usepackage{adjustbox}
1717
\usepackage{siunitx}
18+
\usepackage{makecell}
19+
\providecommand{\tightlist}{%
20+
\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
1821

1922
% Fix any Unicode issues
2023
\DeclareUnicodeCharacter{2212}{-}
2124
\DeclareUnicodeCharacter{2264}{\leq}
2225
\DeclareUnicodeCharacter{2265}{\geq}
26+
\definecolor{bestgreen}{RGB}{0,150,0}
27+
\definecolor{worstred}{RGB}{200,0,0}
2328

2429
\title{QQN: A Quadratic Hybridization of Quasi-Newton Methods for Nonlinear Optimization}
2530
\author{Andrew Charneski\\SimiaCryptus Software}

‎src/analysis/plotting.rs‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ impl PlottingEngine {
340340
}
341341

342342
// Color palette for different optimizers
343-
let colors = [
343+
let colors: [&RGBColor; 24] = [
344344
&RED,
345345
&BLUE,
346346
&GREEN,
@@ -372,13 +372,13 @@ impl PlottingEngine {
372372
for (optimizer_idx, optimizer_name) in unique_optimizers.iter().enumerate() {
373373
let color = colors[optimizer_idx % colors.len()];
374374
let traces_for_optimizer = &optimizer_traces[optimizer_name];
375-
debug!(
376-
"Drawing {} traces for optimizer: {}",
375+
info!(
376+
"Drawing {} traces/runs for optimizer: {}",
377377
traces_for_optimizer.len(),
378378
optimizer_name
379379
);
380380

381-
for trace in traces_for_optimizer {
381+
for (run_idx, trace) in traces_for_optimizer.iter().enumerate() {
382382
let series_data: Vec<(usize, f64)> = trace
383383
.evaluation_counts
384384
.iter()
@@ -390,15 +390,23 @@ impl PlottingEngine {
390390
debug!("Skipping empty series for {}", optimizer_name);
391391
continue;
392392
}
393-
debug!(
394-
"Drawing series with {} data points for {}",
393+
info!(
394+
"Drawing series with {} data points for {} (run {})",
395395
series_data.len(),
396-
optimizer_name
396+
optimizer_name,
397+
run_idx
397398
);
399+
// Use slightly different line styles for multiple runs of the same optimizer
400+
let line_style = if run_idx == 0 {
401+
(*color).to_rgba()
402+
} else {
403+
// Make subsequent runs more transparent based on run index
404+
(*color).mix(0.8 - (run_idx as f64 * 0.05).min(0.3)).to_rgba()
405+
};
398406

399407
// Draw series
400408
chart
401-
.draw_series(LineSeries::new(series_data.clone(), color))
409+
.draw_series(LineSeries::new(series_data.clone(), &line_style))
402410
.map_err(|e| anyhow::anyhow!("Series drawing error: {}", e))?;
403411
// Add markers at regular intervals for better visibility
404412
let marker_interval = series_data.len().max(1) / 20 + 1;
@@ -408,7 +416,7 @@ impl PlottingEngine {
408416
series_data
409417
.iter()
410418
.step_by(marker_interval)
411-
.map(|&(x, y)| Circle::new((x, y), 3, color.filled())),
419+
.map(|&(x, y)| Circle::new((x, y), 3, line_style.filled())),
412420
)
413421
.map_err(|e| anyhow::anyhow!("Marker drawing error: {}", e))?;
414422
}
@@ -606,7 +614,7 @@ impl PlottingEngine {
606614
optimizer_name
607615
);
608616

609-
for trace in traces_for_optimizer {
617+
for (run_idx, trace) in traces_for_optimizer.iter().enumerate() {
610618
let series_data: Vec<(usize, f64)> = trace
611619
.evaluation_counts
612620
.iter()
@@ -622,11 +630,18 @@ impl PlottingEngine {
622630
// Only draw if we have valid data points
623631
if !series_data.is_empty() {
624632
debug!(
625-
"Drawing log series with {} data points for {}",
633+
"Drawing log series with {} data points for {} (run {})",
626634
series_data.len(),
627-
optimizer_name
635+
optimizer_name,
636+
run_idx
628637
);
629-
chart.draw_series(LineSeries::new(series_data.clone(), color))?;
638+
// Use slightly different line styles for multiple runs of the same optimizer
639+
let line_style = if run_idx == 0 {
640+
(*color).to_rgba()
641+
} else {
642+
// Make subsequent runs slightly more transparent
643+
(*color).mix(0.7).to_rgba()
644+
};
630645

631646
// Add markers for better visibility
632647
let marker_interval = series_data.len().max(1) / 20 + 1;
@@ -635,7 +650,7 @@ impl PlottingEngine {
635650
series_data
636651
.iter()
637652
.step_by(marker_interval)
638-
.map(|&(x, y)| Circle::new((x, y), 3, color.filled())),
653+
.map(|&(x, y)| Circle::new((x, y), 3, line_style.filled())),
639654
)?;
640655
} else {
641656
debug!("Skipping empty log series for {}", optimizer_name);
@@ -1487,4 +1502,4 @@ mod tests {
14871502
assert!(data.q1 < data.median);
14881503
assert!(data.median < data.q3);
14891504
}
1490-
}
1505+
}

‎src/benchmarks/evaluation.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,12 @@ impl BenchmarkRunner {
331331
mut optimizers: Vec<Box<dyn Optimizer>>,
332332
) -> Result<BenchmarkResults, BenchmarkError> {
333333
let mut results = BenchmarkResults::new(self.config.clone());
334+
info!(
335+
"Running benchmarks with {} problems, {} optimizers, {} runs each",
336+
problems.len(),
337+
optimizers.len(),
338+
self.config.num_runs
339+
);
334340

335341
for problem in &problems {
336342
for optimizer in &mut optimizers {
@@ -349,6 +355,10 @@ impl BenchmarkRunner {
349355
}
350356
}
351357
}
358+
info!(
359+
"Benchmark complete: collected {} total results",
360+
results.results.len()
361+
);
352362

353363
Ok(results)
354364
}
@@ -1086,4 +1096,4 @@ impl ProblemSpec {
10861096
.clone()
10871097
.unwrap_or_else(|| self.problem.name().to_string())
10881098
}
1089-
}
1099+
}

‎src/experiment_runner/plotting_manager.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ impl PlottingManager {
4040
// Generate convergence plots for each problem
4141
for (problem, results) in all_results {
4242
let problem_name = problem.get_name();
43+
info!(
44+
"Processing problem '{}' with {} total results",
45+
problem_name,
46+
results.results.len()
47+
);
4348
let traces: Vec<ExtendedOptimizationTrace> = results
4449
.results
4550
.iter()
@@ -63,7 +68,7 @@ impl PlottingManager {
6368

6469
if !traces.is_empty() {
6570
info!(
66-
"Generating plots for {} with {} optimizers",
71+
"Generating plots for {} with {} total traces",
6772
problem_name,
6873
traces.len()
6974
);
@@ -143,4 +148,4 @@ impl PlottingManager {
143148
}
144149
}
145150
}
146-
}
151+
}

‎tests/benchmark_reports.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ async fn calibration() -> Result<(), Box<dyn Error + Send + Sync>> {
3838

3939
#[tokio::test]
4040
async fn full_test() -> Result<(), Box<dyn Error + Send + Sync>> {
41-
// init_logging(false)?;
41+
init_logging(false)?;
4242
// Disable no threshold mode for this test
4343
disable_no_threshold_mode();
4444

@@ -64,9 +64,9 @@ async fn test_all(
6464
prefix: &str,
6565
problems: Vec<ProblemSpec>,
6666
) -> Result<(), Box<dyn Error + Send + Sync>> {
67-
let max_evals = 10000;
68-
let num_runs = 10;
69-
let max_cpu = Some(8);
67+
let max_evals = 100;
68+
let num_runs = 5;
69+
let max_cpu = Some(1);
7070
run_benchmark(
7171
&format!("{prefix}all_optimizers_"),
7272
max_evals,

0 commit comments

Comments
 (0)