Merge pull request #1386 from pints-team/func-test-organising

Re-organisation/tidying of functional tests submodule

pints/functionaltests/__init__.py

@@ -1,13 +1,43 @@
 #
+# "Functional test" module for PINTS.
+#
 # This file is part of PINTS (https://github.com/pints-team/pints/) which is
 # released under the BSD 3-clause license. See accompanying LICENSE.md for
 # copyright notice and full license details.
 #
 
-from .differential_evolution import test_differential_evolution_on_banana
-from .differential_evolution import test_differential_evolution_on_two_dim_gaussian
+# Import all problem classes straight into this module, so that they can be
+# addressed as e.g. pints.functionaltests.RunMcmcMethodOnAnnulus.
+from ._problems import (    # noqa
+    RunMcmcMethodOnAnnulus,
+    RunMcmcMethodOnBanana,
+    RunMcmcMethodOnCone,
+    RunMcmcMethodOnCorrelatedGaussian,
+    RunMcmcMethodOnHighDimensionalGaussian,
+    RunMcmcMethodOnMultimodalGaussian,
+    RunMcmcMethodOnTwoDimGaussian,
+)
+
+# Import all test modules (not methods!) directly into this method, so that
+# they can be addressed as e.g.
+# pints.functionaltests.dram_acmc.two_dim_gaussian().
+from . import (     # noqa
+    differential_evolution_mcmc,
+    dram_acmc,
+    dream_mcmc,
+    emcee_hammer_mcmc,
+    haario_acmc,
+    haario_bardenet_acmc,
+    hamiltonian_mcmc,
+    mala_mcmc,
+    metropolis_random_walk_mcmc,
+    monomial_gamma_hamiltonian_mcmc,
+    no_u_turn_mcmc,
+    population_mcmc,
+    relativistic_mcmc,
+    slice_stepout_mcmc,
+)
+
 
-from .haario_bardenet_acmc import test_haario_bardenet_acmc_on_annulus
-from .haario_bardenet_acmc import test_haario_bardenet_acmc_on_banana
-from .haario_bardenet_acmc import test_haario_bardenet_acmc_on_correlated_gaussian
-from .haario_bardenet_acmc import test_haario_bardenet_acmc_on_two_dim_gaussian
+# Test discovery methods
+from ._discovery import tests

pints/functionaltests/_discovery.py

@@ -0,0 +1,40 @@
+#
+# Functional test discovery methods for PINTS.
+#
+# This file is part of PINTS (https://github.com/pints-team/pints/) which is
+# released under the BSD 3-clause license. See accompanying LICENSE.md for
+# copyright notice and full license details.
+#
+import inspect
+
+import pints.functionaltests as ft
+
+
+def tests(method=None):
+    """
+    Returns a list of all functional tests, each represented as a tuple
+    ``(method, test)`` where ``method`` is the PINTS class being tested and
+    ``test`` is a callable that returns the test results.
+
+    If the optional argument ``method`` is given, only tests for this method
+    are returned.
+    """
+    # Get all modules imported into this module
+    modules = [getattr(ft, x) for x in dir(ft) if not x.startswith('_')]
+    modules = [x for x in modules if inspect.ismodule(x)]
+
+    # Look for (explicitly defined) tests
+    tests = []
+    for module in modules:
+        try:
+            m_method = module._method
+            m_tests = module._functional_tests
+        except AttributeError:
+            continue
+
+        if method is None or method == m_method:
+            for test in m_tests:
+                tests.append((m_method, test))
+
+    return tests
+

pints/functionaltests/_problems.py

@@ -1,15 +1,44 @@
 #
+# Shared problems used in functional testing.
+#
 # This file is part of PINTS (https://github.com/pints-team/pints/) which is
 # released under the BSD 3-clause license. See accompanying LICENSE.md for
 # copyright notice and full license details.
 #
-
 import numpy as np
+
 import pints
 import pints.toy
 
 
 class RunMcmcMethodOnProblem(object):
+    """
+    Base class for tests that run an MCMC method on a log-PDF.
+
+    Parameters
+    ----------
+    log_pdf : pints.LogPDF
+        The PDF to sample. Will be passed to a :class:`pints.MCMCController`.
+    x0
+        One or more starting points to be passed to the
+        :class:`pints.MCMCController`.
+    sigma0
+        One or more ``sigma0`` parameters to be passed to the
+        :class:`pints.MCMCController`.
+    method : pints.MCMCSampler
+        The method to test. Will be passed to the
+        :class:`pints.MCMCController`.
+    n_chains : int
+        The number of chains to run. Will be passed to the
+        :class:`pints.MCMCController`.
+    n_iterations : int
+        The number of iterations to run
+    n_warmup : int
+        The number of iterations to discard
+    method_hyper_parameters : list
+        A list of hyperparameter values.
+
+    """
 
     def __init__(self, log_pdf, x0, sigma0, method, n_chains, n_iterations,
                  n_warmup, method_hyper_parameters):
@@ -26,9 +55,11 @@ def __init__(self, log_pdf, x0, sigma0, method, n_chains, n_iterations,
     def estimate_kld(self):
         """
         Estimates the Kullback-Leibler divergence.
+
+        Raises an ``AttributeError`` if the underlying LogPDF does not have a
+        method ``kl_divergence()``.
         """
         chains = np.vstack(self.chains)
-
         return np.mean(self.log_pdf.kl_divergence(chains))
 
     def estimate_mean_ess(self):
@@ -47,15 +78,21 @@ def estimate_mean_ess(self):
 
     def estimate_distance(self):
         """
-        Estimates a measure of distance for the `pints.AnnulusLogPDF` class.
+        Estimates a measure of distance between the sampled chains and the true
+        distribution.
+
+        Raises an ``AttributeError`` if the underlying LogPDF does not have a
+        method ``distance()``.
         """
         return self.log_pdf.distance(np.vstack(self.chains))
 
 
 class RunMcmcMethodOnTwoDimGaussian(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on a standard 2 dimensional Gaussian
-    distribution.
+    Tests a given MCMC method on a two-dimensional Gaussian distribution with
+    means ``[0, 0]`` and sigma ``[1, 1]``.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
 
     def __init__(self, method, n_chains, n_iterations, n_warmup,
@@ -75,7 +112,11 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
 class RunMcmcMethodOnBanana(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `pints.toy.TwistedGaussianLogPDF`.
+    Tests a given MCMC method on a two-dimensional
+    :class:`pints.toy.TwistedGaussianLogPDF` distribution with means
+    ``[0, 0]``.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):
@@ -91,6 +132,7 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
                          n_warmup, method_hyper_parameters)
 
 
+'''
 class RunMcmcMethodOnSimpleEggBox(RunMcmcMethodOnProblem):
     """
     Tests a given MCMC method on `pints.toy.SimpleEggBoxLogPDF`.
@@ -105,11 +147,15 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
         super().__init__(log_pdf, x0, sigma0, method, n_chains, n_iterations,
                          n_warmup, method_hyper_parameters)
+'''
 
 
 class RunMcmcMethodOnHighDimensionalGaussian(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `pints.toy.HighDimensionalGaussianLogPDF`.
+    Tests a given MCMC method on a 20-dimensional
+    :class:`pints.toy.HighDimensionalGaussianLogPDF` centered at the origin.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):
@@ -123,13 +169,14 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
 class RunMcmcMethodOnCorrelatedGaussian(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `pints.toy.HighDimensionalGaussianLogPDF`
-    but using a 6-dimensional problem with higher correlation.
+    Tests a given MCMC method on a 6-dimensional, highly correlated
+    :class:`pints.toy.HighDimensionalGaussianLogPDF` centered at the origin.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):
-        log_pdf = pints.toy.HighDimensionalGaussianLogPDF(
-            dimension=6, rho=0.8)
+        log_pdf = pints.toy.HighDimensionalGaussianLogPDF(dimension=6, rho=0.8)
         x0 = np.random.uniform(-4, 4, size=(n_chains, 6))
         sigma0 = None
 
@@ -139,7 +186,11 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
 class RunMcmcMethodOnAnnulus(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `pints.AnnulusLogPDF`.
+    Tests a given MCMC method on a two-dimensional
+    :class:`pints.toy.AnnulusLogPDF` distribution, with its highest values at
+    any point ``x`` with ``np.linalg.norm(x) == 10``.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):
@@ -153,17 +204,21 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
 class RunMcmcMethodOnMultimodalGaussian(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `MultimodalGaussianLogPDF`.
+    Tests a given MCMC method on a two-dimensional
+    :class:`pints.toy.MultimodalGaussianLogPDF` with modes at ``[0, 0]``,
+    ``[5, 10]``, and ``[10, 0]``.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):
+        modes = [[0, 0],
+                 [5, 10],
+                 [10, 0]]
         covariances = [[[1, 0], [0, 1]],
                        [[2, 0.8], [0.8, 3]],
                        [[1, -0.5], [-0.5, 1]]]
-        log_pdf = pints.toy.MultimodalGaussianLogPDF(modes=[[0, 0],
-                                                            [5, 10],
-                                                            [10, 0]],
-                                                     covariances=covariances)
+        log_pdf = pints.toy.MultimodalGaussianLogPDF(modes, covariances)
         x0 = log_pdf.sample(n_chains)
         sigma0 = None
 
@@ -173,7 +228,10 @@ def __init__(self, method, n_chains, n_iterations, n_warmup,
 
 class RunMcmcMethodOnCone(RunMcmcMethodOnProblem):
     """
-    Tests a given MCMC method on `MultimodalGaussianLogPDF`.
+    Tests a given MCMC method on a two-dimensional
+    :class:`pints.toy,ConeLogPDF` centered at ``[0, 0]``.
+
+    For constructor arguments, see :class:`RunMcmcMethodOnProblem`.
     """
     def __init__(self, method, n_chains, n_iterations, n_warmup,
                  method_hyper_parameters=None):