CMA-ES

Changelog.md

@@ -5,6 +5,12 @@ All notable Changes to the Julia package `Manopt.jl` will be documented in this
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.59] - unreleased
+
+### Added
+
+* A Riemannian variant of the CMA-ES (Covariance Matrix Adaptation Evolutionary Strategy) algorithm, `cma_es`.
+
 ## [0.4.58] - March 18, 2024
 
 ### Added

docs/make.jl

@@ -170,6 +170,7 @@ makedocs(;
             "Alternating Gradient Descent" => "solvers/alternating_gradient_descent.md",
             "Augmented Lagrangian Method" => "solvers/augmented_Lagrangian_method.md",
             "Chambolle-Pock" => "solvers/ChambollePock.md",
+            "CMA-ES" => "solvers/cma_es.md",
             "Conjugate gradient descent" => "solvers/conjugate_gradient_descent.md",
             "Convex bundle method" => "solvers/convex_bundle_method.md",
             "Cyclic Proximal Point" => "solvers/cyclic_proximal_point.md",

docs/src/references.bib

@@ -412,6 +412,16 @@ @article{HagerZhang:2006
     VOLUME       = {2},
     YEAR         = {2006}
 }
+
+@misc{Hansen:2023,
+	title = {The {CMA} {Evolution} {Strategy}: {A} {Tutorial}},
+	shorttitle = {The {CMA} {Evolution} {Strategy}},
+	url = {http://arxiv.org/abs/1604.00772},
+	doi = {10.48550/arXiv.1604.00772},
+	author = {Hansen, Nikolaus},
+	month = mar,
+	year = {2023},
+}
 @article{HestenesStiefel:1952,
     AUTHOR    = {M.R. Hestenes and E. Stiefel},
     DOI       = {10.6028/jres.049.044},

docs/src/solvers/cma_es.md

@@ -0,0 +1,29 @@
+# Particle swarm optimization
+
+```@meta
+CurrentModule = Manopt
+```
+
+```@docs
+cma_es
+
+```
+
+## State
+
+```@docs
+CMAESState
+```
+
+## Stopping Criteria
+
+```@docs
+CMAESConditionCov
+```
+
+## Literature
+
+```@bibliography
+Pages = ["particle_swarm.md"]
+Canonical=false
+```

src/Manopt.jl

@@ -18,7 +18,7 @@ using Colors
 using DataStructures: CircularBuffer, capacity, length, push!, size, isfull
 using Dates: Millisecond, Nanosecond, Period, canonicalize, value
 using LinearAlgebra:
-    Diagonal, I, eigen, eigvals, tril, Symmetric, dot, cholesky, eigmin, opnorm
+    cond, Diagonal, I, eigen, eigvals, tril, Symmetric, dot, cholesky, eigmin, opnorm, mul!
 using ManifoldDiff:
     adjoint_differential_log_argument,
     adjoint_differential_log_argument!,
@@ -93,6 +93,7 @@ using ManifoldsBase:
     inner,
     inverse_retract,
     inverse_retract!,
+    is_flat,
     is_point,
     is_vector,
     log,
@@ -102,6 +103,7 @@ using ManifoldsBase:
     mid_point!,
     norm,
     number_eltype,
+    number_of_coordinates,
     power_dimensions,
     project,
     project!,
@@ -128,7 +130,7 @@ using Markdown
 using Preferences:
     @load_preference, @set_preferences!, @has_preference, @delete_preferences!
 using Printf
-using Random: shuffle!, rand, randperm
+using Random: AbstractRNG, default_rng, shuffle!, rand, randn!, randperm
 using Requires
 using SparseArrays
 using Statistics: cor, cov, mean, std
@@ -142,6 +144,7 @@ include("solvers/alternating_gradient_descent.jl")
 include("solvers/augmented_Lagrangian_method.jl")
 include("solvers/convex_bundle_method.jl")
 include("solvers/ChambollePock.jl")
+include("solvers/cma_es.jl")
 include("solvers/conjugate_gradient_descent.jl")
 include("solvers/cyclic_proximal_point.jl")
 include("solvers/difference_of_convex_algorithm.jl")
@@ -416,6 +419,7 @@ export adaptive_regularization_with_cubics,
     convex_bundle_method!,
     ChambollePock,
     ChambollePock!,
+    cma_es,
     conjugate_gradient_descent,
     conjugate_gradient_descent!,
     cyclic_proximal_point,

src/plans/stopping_criterion.jl

@@ -957,6 +957,8 @@ function Base.:|(s1::StopWhenAny, s2::T) where {T<:StoppingCriterion}
     return StopWhenAny(s1.criteria..., s2)
 end
 
+is_active_stopping_criterion(c::StoppingCriterion) = !isempty(c.reason)
+
 @doc raw"""
     get_active_stopping_criteria(c)
 
@@ -974,7 +976,7 @@ end
 # for non-array containing stopping criteria, the recursion ends in either
 # returning nothing or an 1-element array containing itself
 function get_active_stopping_criteria(c::sC) where {sC<:StoppingCriterion}
-    if c.reason != ""
+    if is_active_stopping_criterion(c)
         return [c] # recursion top
     else
         return []