handle multiple noisy instances

DESCRIPTION

@@ -61,5 +61,5 @@ LazyData: yes
 Encoding: UTF-8
 ByteCompile: yes
 Version: 1.1.3
-RoxygenNote: 6.0.1
+RoxygenNote: 6.1.1
 VignetteBuilder: knitr

NAMESPACE

@@ -58,6 +58,7 @@ export(renderExampleRunPlot)
 export(setMBOControlInfill)
 export(setMBOControlMultiObj)
 export(setMBOControlMultiPoint)
+export(setMBOControlNoisy)
 export(setMBOControlTermination)
 export(trafoLog)
 export(trafoSqrt)

R/SMBO.R

@@ -49,6 +49,8 @@ initSMBO = function(par.set, design, learner = NULL, control, minimize = rep(TRU
 #'   Outcome of the optimization.
 #'   For multiple results use a list.
 #'   For a result of a multi-objective function use a numeric vector.
+#'   For multiple results of for noisy instances use a list.
+#'   Each list element should correspond to one x value.
 #'
 #' @return [\code{\link{OptState}}]
 #' @export

R/evalTargetFun.R

@@ -25,36 +25,79 @@ evalTargetFun.OptState = function(opt.state, xs, extras) {
   # short names and so on
   nevals = length(xs)
   ny = control$n.objectives
-  num.format = control$output.num.format
-  num.format.string = paste("%s = ", num.format, sep = "")
-  dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer")
   imputeY = control$impute.y.fun
 
   # trafo X points
   xs.trafo = lapply(xs, trafoValue, par = par.set)
 
   # function to measure of fun call
-    wrapFun = function(x) {
-      st = proc.time()
-      y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem)))
-      user.extras = list()
+  wrapFun = function(x) {
+    st = proc.time()
+    y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem)))
+    user.extras = list()
       # here we extract additional stuff which the user wants to log in the opt path
-      if (hasAttributes(y, "extras")) {
-        user.extras = attr(y, "extras")
-        y = setAttribute(y, "extras", NULL)
-      }
-      st = proc.time() - st
-      list(y = y, time = st[3], user.extras = user.extras)
+    if (hasAttributes(y, "extras")) {
+      user.extras = attr(y, "extras")
+      y = setAttribute(y, "extras", NULL)
     }
+    if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) {
+      user.extras = c(user.extras, x[control$noisy.instance.param])
+    }
+    st = proc.time() - st
+    list(y = y, time = st[3], user.extras = user.extras)
+  }
 
   # do we have a valid y object?
   isYValid = function(y) {
-    !is.error(y) && testNumeric(y, len = ny, any.missing = FALSE, finite = TRUE)
+    if (!isTRUE(control$noisy.self.replicating)) {
+      len = NULL
+    } else {
+      len = ny
+    }
+    !is.error(y) && testNumeric(y, len = len, any.missing = FALSE, finite = TRUE)
   }
 
   # return error objects if we impute
-  res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval",
-    impute.error = if (is.null(imputeY)) NULL else identity)
+  res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", impute.error = if (is.null(imputeY)) NULL else identity)
+
+  # handle noisy instances
+  if (isTRUE(control$noisy.self.replicating)) {
+    nevals.each = lengths(extractSubList(res, "y", simplify = FALSE))
+    nevals = sum(nevals.each)
+
+    # replications for opt path stuff
+    repVec = function(x, fun = replicate) {
+      unlist(Map(fun, nevals.each, x, simplify = FALSE), recursive = FALSE)
+    }
+    xs = repVec(xs)
+    xs.trafo = repVec(xs.trafo)
+
+    #set extras to NA that are only important for the first point
+    setNAfun = function(n, xs, ...) {
+      res = replicate(n = n, expr = xs, ...)
+      res[-1] = lapply(res[-1], function(x) {
+        x[c("train.time", "error.model", "propose.time")] = NA
+        x
+      })
+      res
+    }
+    extras = repVec(extras, setNAfun)
+
+    # handle result list
+    res = lapply(res, function(r) {
+      if (is.error(r)) {
+        rep(list(r), control$noisy.instances)
+      } else {
+        lapply(seq_along(r$y), function(i) {
+          list(y = r$y[i], time = r$time / length(r$y), user.extras = c(r$user.extras, setNames(list(i), control$noisy.instance.param)))
+        })
+      }
+    })
+    res = unlist(res, recursive = FALSE)
+  }
+
+  num.format.string = paste("%s = ", control$output.num.format, sep = "")
+  dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer")
 
   # loop evals and to some post-processing
   for (i in seq_len(nevals)) {
@@ -92,7 +135,7 @@ evalTargetFun.OptState = function(opt.state, xs, extras) {
     # showInfo - use the trafo'd value here!
     showInfo(getOptProblemShowInfo(opt.problem), "[mbo] %i: %s : %s : %.1f secs%s : %s",
       dob,
-      paramValueToString(par.set, x.trafo, num.format = num.format),
+      paramValueToString(par.set, x.trafo, num.format = control$output.num.format),
       collapse(sprintf(num.format.string, control$y.name, y2), ", "),
       ytime,
       ifelse(y.valid, "", " (imputed)"),

R/setMBOControlNoisy.R

@@ -0,0 +1,18 @@
+#' @title Set multipoint proposal options.
+#' @description
+#' Extends an MBO control object with options for multipoint proposal.
+#' @template arg_control
+#' @param self.replicating [\code{logical(1)}]\cr
+#'   TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect.
+#' @return [\code{\link{MBOControl}}].
+#' @family MBOControl
+#' @export
+setMBOControlNoisy = function(control, self.replicating) {
+
+  assertClass(control, "MBOControl")
+
+  control$noisy.self.replicating = assertFlag(self.replicating %??% control$noisy.self.replicating %??% TRUE, na.ok = FALSE)
+  control$noisy.instance.param = "noisy.repl"
+
+  return(control)
+}

tests/testthat/test_mbo_km.R

@@ -1,6 +1,6 @@
-context("mbo km")
+context("mbo noisy")
 
-test_that("mbo works with km", {
+test_that("mbo works with multiple instances of noisy problems", {
   des = testd.fsphere.2d
   des$y = apply(des, 1, testf.fsphere.2d)
   learner = makeLearner("regr.km", nugget.estim = TRUE)

tests/testthat/test_mbo_noisy.R

@@ -0,0 +1,77 @@
+context("mbo noisy")
+
+test_that("mbo works with self replicating instances of noisy problems", {
+  ps = makeNumericParamSet("x", 1, -7, 7)
+  fun = smoof::makeSingleObjectiveFunction(
+    fn = function(x) x^2 + rnorm(5, 0.01),
+    par.set = ps,
+    noisy = TRUE
+  )
+  ctrl = makeMBOControl()
+  ctrl = setMBOControlTermination(ctrl, iters = 5L)
+  ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
+  ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
+  or = mbo(fun, control = ctrl)
+  opdf = as.data.frame(or$opt.path)
+  expect_true(all(opdf$noisy.repl %in% 1:5))
+  expect_true(all(table(opdf$x) == 5))
+
+  # now the function has varying n results
+  i = 0
+  fun = smoof::makeSingleObjectiveFunction(
+    fn = function(x) {
+      i <<- i + 1
+      x^2 + rnorm(i, 0.01)
+    },
+    par.set = ps,
+    noisy = TRUE
+  )
+  ctrl = makeMBOControl()
+  ctrl = setMBOControlTermination(ctrl, iters = 2L)
+  ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
+  ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
+  or = mbo(fun, control = ctrl)
+  opdf = as.data.frame(or$opt.path)
+  expect_true(all(opdf$noisy.repl == unlist(lapply(1:6, function(x) head(1:6,x)))))
+
+  # returns the right result
+  ps = makeNumericParamSet("x", 2, -7, 7)
+  fun = smoof::makeSingleObjectiveFunction(
+    fn = function(x) {
+      x = sum(unlist(x))
+      res = x^2 + rnorm(5, 0.01)
+      if (abs(x)>5) res[1] = -10
+      return(res)
+    },
+    par.set = ps,
+    noisy = TRUE
+  )
+  ctrl = makeMBOControl(final.method = "best.predicted")
+  ctrl = setMBOControlTermination(ctrl, iters = 5L)
+  ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
+  ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
+  or = mbo(fun, control = ctrl)
+  expect_true(abs(sum(or$x$x))<5)
+
+  # deals with errors in noisy repls
+  if (FALSE) {
+    ps = makeNumericParamSet("x", 2, -7, 7)
+    fun = smoof::makeSingleObjectiveFunction(
+      fn = function(x) {
+        x = sum(unlist(x))
+        res = x^2 + rnorm(5, 0.01)
+        res = as.list(res)
+        res[[1]] = base::simpleError("Something went wrong!")
+      },
+      par.set = ps,
+      noisy = TRUE
+    )
+    ctrl = makeMBOControl(final.method = "best.predicted", impute.y.fun = function(x, y, opt.path) 0, on.surrogate.error = "warn")
+    ctrl = setMBOControlTermination(ctrl, iters = 5L)
+    ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
+    ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
+    or = mbo(fun, control = ctrl)
+    as.data.frame(or$opt.path)
+  }
+})
+