diff --git a/DESCRIPTION b/DESCRIPTION index 55d0869ea..16e8a09b6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -61,5 +61,5 @@ LazyData: yes Encoding: UTF-8 ByteCompile: yes Version: 1.1.3 -RoxygenNote: 6.0.1 +RoxygenNote: 6.1.1 VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index 390bc12d5..df1154ad4 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -58,6 +58,7 @@ export(renderExampleRunPlot) export(setMBOControlInfill) export(setMBOControlMultiObj) export(setMBOControlMultiPoint) +export(setMBOControlNoisy) export(setMBOControlTermination) export(trafoLog) export(trafoSqrt) diff --git a/R/SMBO.R b/R/SMBO.R index 89caad62f..0839efaa5 100644 --- a/R/SMBO.R +++ b/R/SMBO.R @@ -49,6 +49,8 @@ initSMBO = function(par.set, design, learner = NULL, control, minimize = rep(TRU #' Outcome of the optimization. #' For multiple results use a list. #' For a result of a multi-objective function use a numeric vector. +#' For multiple results of for noisy instances use a list. +#' Each list element should correspond to one x value. #' #' @return [\code{\link{OptState}}] #' @export diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R index 3f79fb118..d9deb9d59 100644 --- a/R/evalTargetFun.R +++ b/R/evalTargetFun.R @@ -25,36 +25,79 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { # short names and so on nevals = length(xs) ny = control$n.objectives - num.format = control$output.num.format - num.format.string = paste("%s = ", num.format, sep = "") - dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer") imputeY = control$impute.y.fun # trafo X points xs.trafo = lapply(xs, trafoValue, par = par.set) # function to measure of fun call - wrapFun = function(x) { - st = proc.time() - y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem))) - user.extras = list() + wrapFun = function(x) { + st = proc.time() + y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem))) + user.extras = list() # here we extract additional stuff which the user wants to log in the opt path - if (hasAttributes(y, "extras")) { - user.extras = attr(y, "extras") - y = setAttribute(y, "extras", NULL) - } - st = proc.time() - st - list(y = y, time = st[3], user.extras = user.extras) + if (hasAttributes(y, "extras")) { + user.extras = attr(y, "extras") + y = setAttribute(y, "extras", NULL) } + if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) { + user.extras = c(user.extras, x[control$noisy.instance.param]) + } + st = proc.time() - st + list(y = y, time = st[3], user.extras = user.extras) + } # do we have a valid y object? isYValid = function(y) { - !is.error(y) && testNumeric(y, len = ny, any.missing = FALSE, finite = TRUE) + if (!isTRUE(control$noisy.self.replicating)) { + len = NULL + } else { + len = ny + } + !is.error(y) && testNumeric(y, len = len, any.missing = FALSE, finite = TRUE) } # return error objects if we impute - res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", - impute.error = if (is.null(imputeY)) NULL else identity) + res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", impute.error = if (is.null(imputeY)) NULL else identity) + + # handle noisy instances + if (isTRUE(control$noisy.self.replicating)) { + nevals.each = lengths(extractSubList(res, "y", simplify = FALSE)) + nevals = sum(nevals.each) + + # replications for opt path stuff + repVec = function(x, fun = replicate) { + unlist(Map(fun, nevals.each, x, simplify = FALSE), recursive = FALSE) + } + xs = repVec(xs) + xs.trafo = repVec(xs.trafo) + + #set extras to NA that are only important for the first point + setNAfun = function(n, xs, ...) { + res = replicate(n = n, expr = xs, ...) + res[-1] = lapply(res[-1], function(x) { + x[c("train.time", "error.model", "propose.time")] = NA + x + }) + res + } + extras = repVec(extras, setNAfun) + + # handle result list + res = lapply(res, function(r) { + if (is.error(r)) { + rep(list(r), control$noisy.instances) + } else { + lapply(seq_along(r$y), function(i) { + list(y = r$y[i], time = r$time / length(r$y), user.extras = c(r$user.extras, setNames(list(i), control$noisy.instance.param))) + }) + } + }) + res = unlist(res, recursive = FALSE) + } + + num.format.string = paste("%s = ", control$output.num.format, sep = "") + dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer") # loop evals and to some post-processing for (i in seq_len(nevals)) { @@ -92,7 +135,7 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { # showInfo - use the trafo'd value here! showInfo(getOptProblemShowInfo(opt.problem), "[mbo] %i: %s : %s : %.1f secs%s : %s", dob, - paramValueToString(par.set, x.trafo, num.format = num.format), + paramValueToString(par.set, x.trafo, num.format = control$output.num.format), collapse(sprintf(num.format.string, control$y.name, y2), ", "), ytime, ifelse(y.valid, "", " (imputed)"), diff --git a/R/setMBOControlNoisy.R b/R/setMBOControlNoisy.R new file mode 100644 index 000000000..91383f81b --- /dev/null +++ b/R/setMBOControlNoisy.R @@ -0,0 +1,18 @@ +#' @title Set multipoint proposal options. +#' @description +#' Extends an MBO control object with options for multipoint proposal. +#' @template arg_control +#' @param self.replicating [\code{logical(1)}]\cr +#' TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect. +#' @return [\code{\link{MBOControl}}]. +#' @family MBOControl +#' @export +setMBOControlNoisy = function(control, self.replicating) { + + assertClass(control, "MBOControl") + + control$noisy.self.replicating = assertFlag(self.replicating %??% control$noisy.self.replicating %??% TRUE, na.ok = FALSE) + control$noisy.instance.param = "noisy.repl" + + return(control) +} diff --git a/man/makeMBOControl.Rd b/man/makeMBOControl.Rd index 55847a818..a93f06d45 100644 --- a/man/makeMBOControl.Rd +++ b/man/makeMBOControl.Rd @@ -7,10 +7,10 @@ \usage{ makeMBOControl(n.objectives = 1L, propose.points = 1L, final.method = "best.true.y", final.evals = 0L, y.name = "y", - impute.y.fun = NULL, trafo.y.fun = NULL, suppress.eval.errors = TRUE, - save.on.disk.at = integer(0L), save.on.disk.at.time = Inf, - save.file.path = file.path(getwd(), "mlr_run.RData"), - store.model.at = NULL, resample.at = integer(0), + impute.y.fun = NULL, trafo.y.fun = NULL, + suppress.eval.errors = TRUE, save.on.disk.at = integer(0L), + save.on.disk.at.time = Inf, save.file.path = file.path(getwd(), + "mlr_run.RData"), store.model.at = NULL, resample.at = integer(0), resample.desc = makeResampleDesc("CV", iter = 10), resample.measures = list(mse), output.num.format = "\%.3g", on.surrogate.error = "stop") @@ -120,5 +120,7 @@ Creates a control object for MBO optimization. Other MBOControl: \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/plotMBOResult.Rd b/man/plotMBOResult.Rd index 54d88ea4b..30c679486 100644 --- a/man/plotMBOResult.Rd +++ b/man/plotMBOResult.Rd @@ -6,11 +6,11 @@ \alias{plot.MBOMultiObjResult} \title{MBO Result Plotting} \usage{ -\method{plot}{MBOSingleObjResult}(x, iters = NULL, pause = interactive(), - ...) +\method{plot}{MBOSingleObjResult}(x, iters = NULL, + pause = interactive(), ...) -\method{plot}{MBOMultiObjResult}(x, iters = NULL, pause = interactive(), - ...) +\method{plot}{MBOMultiObjResult}(x, iters = NULL, + pause = interactive(), ...) } \arguments{ \item{x}{[\code{MBOResult}]\cr diff --git a/man/renderExampleRunPlot.Rd b/man/renderExampleRunPlot.Rd index de888f594..c368bf75f 100644 --- a/man/renderExampleRunPlot.Rd +++ b/man/renderExampleRunPlot.Rd @@ -7,8 +7,8 @@ exampleRunMultiObj objects.} \usage{ renderExampleRunPlot(object, iter, densregion = TRUE, se.factor = 1, single.prop.point.plots = FALSE, xlim = NULL, ylim = NULL, - point.size = 3, line.size = 1, trafo = NULL, colors = c("red", "blue", - "green"), ...) + point.size = 3, line.size = 1, trafo = NULL, colors = c("red", + "blue", "green"), ...) } \arguments{ \item{object}{[\code{function}]\cr diff --git a/man/setMBOControlInfill.Rd b/man/setMBOControlInfill.Rd index b6ccc585a..68f911a83 100644 --- a/man/setMBOControlInfill.Rd +++ b/man/setMBOControlInfill.Rd @@ -4,15 +4,17 @@ \alias{setMBOControlInfill} \title{Extends mbo control object with infill criteria and infill optimizer options.} \usage{ -setMBOControlInfill(control, crit = NULL, interleave.random.points = 0L, - filter.proposed.points = NULL, filter.proposed.points.tol = NULL, - opt = "focussearch", opt.restarts = NULL, opt.focussearch.maxit = NULL, +setMBOControlInfill(control, crit = NULL, + interleave.random.points = 0L, filter.proposed.points = NULL, + filter.proposed.points.tol = NULL, opt = "focussearch", + opt.restarts = NULL, opt.focussearch.maxit = NULL, opt.focussearch.points = NULL, opt.cmaes.control = NULL, opt.ea.maxit = NULL, opt.ea.mu = NULL, opt.ea.sbx.eta = NULL, opt.ea.sbx.p = NULL, opt.ea.pm.eta = NULL, opt.ea.pm.p = NULL, opt.ea.lambda = NULL, opt.nsga2.popsize = NULL, opt.nsga2.generations = NULL, opt.nsga2.cprob = NULL, - opt.nsga2.cdist = NULL, opt.nsga2.mprob = NULL, opt.nsga2.mdist = NULL) + opt.nsga2.cdist = NULL, opt.nsga2.mprob = NULL, + opt.nsga2.mdist = NULL) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr @@ -156,5 +158,7 @@ minimizes that. Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlMultiObj}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlMultiObj.Rd b/man/setMBOControlMultiObj.Rd index 4dfba869b..e13ceaec9 100644 --- a/man/setMBOControlMultiObj.Rd +++ b/man/setMBOControlMultiObj.Rd @@ -119,5 +119,7 @@ et.al. (eds.), IEEE, 2005, ISBN 0-7803-9363-5, pp. 2138-2145 Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlMultiPoint.Rd b/man/setMBOControlMultiPoint.Rd index d939eb10f..232dffbeb 100644 --- a/man/setMBOControlMultiPoint.Rd +++ b/man/setMBOControlMultiPoint.Rd @@ -5,9 +5,10 @@ \title{Set multipoint proposal options.} \usage{ setMBOControlMultiPoint(control, method = NULL, cl.lie = NULL, - moimbo.objective = NULL, moimbo.dist = NULL, moimbo.selection = NULL, - moimbo.maxit = NULL, moimbo.sbx.eta = NULL, moimbo.sbx.p = NULL, - moimbo.pm.eta = NULL, moimbo.pm.p = NULL) + moimbo.objective = NULL, moimbo.dist = NULL, + moimbo.selection = NULL, moimbo.maxit = NULL, + moimbo.sbx.eta = NULL, moimbo.sbx.p = NULL, moimbo.pm.eta = NULL, + moimbo.pm.p = NULL) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr @@ -88,5 +89,7 @@ Extends an MBO control object with options for multipoint proposal. Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlNoisy.Rd b/man/setMBOControlNoisy.Rd new file mode 100644 index 000000000..1e22eed84 --- /dev/null +++ b/man/setMBOControlNoisy.Rd @@ -0,0 +1,29 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/setMBOControlNoisy.R +\name{setMBOControlNoisy} +\alias{setMBOControlNoisy} +\title{Set multipoint proposal options.} +\usage{ +setMBOControlNoisy(control, self.replicating) +} +\arguments{ +\item{control}{[\code{\link{MBOControl}}]\cr +Control object for mbo.} + +\item{self.replicating}{[\code{logical(1)}]\cr +TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect.} +} +\value{ +[\code{\link{MBOControl}}]. +} +\description{ +Extends an MBO control object with options for multipoint proposal. +} +\seealso{ +Other MBOControl: \code{\link{makeMBOControl}}, + \code{\link{setMBOControlInfill}}, + \code{\link{setMBOControlMultiObj}}, + \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlTermination}} +} +\concept{MBOControl} diff --git a/man/setMBOControlTermination.Rd b/man/setMBOControlTermination.Rd index 1774144ae..b9c1ddb39 100644 --- a/man/setMBOControlTermination.Rd +++ b/man/setMBOControlTermination.Rd @@ -83,5 +83,7 @@ print(res) Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, - \code{\link{setMBOControlMultiPoint}} + \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}} } +\concept{MBOControl} diff --git a/man/updateSMBO.Rd b/man/updateSMBO.Rd index 31e2f4313..420166083 100644 --- a/man/updateSMBO.Rd +++ b/man/updateSMBO.Rd @@ -18,7 +18,9 @@ One row per set of x values.} \item{y}{[\code{numeric|list}] Outcome of the optimization. For multiple results use a list. -For a result of a multi-objective function use a numeric vector.} +For a result of a multi-objective function use a numeric vector. +For multiple results of for noisy instances use a list. +Each list element should correspond to one x value.} } \value{ [\code{\link{OptState}}] diff --git a/tests/testthat/test_mbo_km.R b/tests/testthat/test_mbo_km.R index 5a4c431a2..ec1364a7b 100644 --- a/tests/testthat/test_mbo_km.R +++ b/tests/testthat/test_mbo_km.R @@ -1,6 +1,6 @@ -context("mbo km") +context("mbo noisy") -test_that("mbo works with km", { +test_that("mbo works with multiple instances of noisy problems", { des = testd.fsphere.2d des$y = apply(des, 1, testf.fsphere.2d) learner = makeLearner("regr.km", nugget.estim = TRUE) diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R new file mode 100644 index 000000000..85cca7981 --- /dev/null +++ b/tests/testthat/test_mbo_noisy.R @@ -0,0 +1,77 @@ +context("mbo noisy") + +test_that("mbo works with self replicating instances of noisy problems", { + ps = makeNumericParamSet("x", 1, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) x^2 + rnorm(5, 0.01), + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl() + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + opdf = as.data.frame(or$opt.path) + expect_true(all(opdf$noisy.repl %in% 1:5)) + expect_true(all(table(opdf$x) == 5)) + + # now the function has varying n results + i = 0 + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) { + i <<- i + 1 + x^2 + rnorm(i, 0.01) + }, + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl() + ctrl = setMBOControlTermination(ctrl, iters = 2L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + opdf = as.data.frame(or$opt.path) + expect_true(all(opdf$noisy.repl == unlist(lapply(1:6, function(x) head(1:6,x))))) + + # returns the right result + ps = makeNumericParamSet("x", 2, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) { + x = sum(unlist(x)) + res = x^2 + rnorm(5, 0.01) + if (abs(x)>5) res[1] = -10 + return(res) + }, + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl(final.method = "best.predicted") + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + expect_true(abs(sum(or$x$x))<5) + + # deals with errors in noisy repls + if (FALSE) { + ps = makeNumericParamSet("x", 2, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) { + x = sum(unlist(x)) + res = x^2 + rnorm(5, 0.01) + res = as.list(res) + res[[1]] = base::simpleError("Something went wrong!") + }, + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl(final.method = "best.predicted", impute.y.fun = function(x, y, opt.path) 0, on.surrogate.error = "warn") + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + as.data.frame(or$opt.path) + } +}) +