From 6a6db9bdb06825da3cdc6bfcf76bcfd242394ab3 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Thu, 1 Feb 2018 17:43:08 +0100 Subject: [PATCH 1/8] handle multiple noisy instances --- NAMESPACE | 1 + R/evalTargetFun.R | 35 ++++++++++++++++++++-- R/setMBOControlNoisy.R | 30 +++++++++++++++++++ man/makeMBOControl.Rd | 1 + man/setMBOControlInfill.Rd | 1 + man/setMBOControlMultiObj.Rd | 1 + man/setMBOControlMultiPoint.Rd | 1 + man/setMBOControlNoisy.Rd | 31 ++++++++++++++++++++ man/setMBOControlTermination.Rd | 3 +- tests/testthat/test_mbo_km.R | 4 +-- tests/testthat/test_mbo_noisy.R | 51 +++++++++++++++++++++++++++++++++ 11 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 R/setMBOControlNoisy.R create mode 100644 man/setMBOControlNoisy.Rd create mode 100644 tests/testthat/test_mbo_noisy.R diff --git a/NAMESPACE b/NAMESPACE index e5a32f857..242f4287b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -56,6 +56,7 @@ export(renderExampleRunPlot) export(setMBOControlInfill) export(setMBOControlMultiObj) export(setMBOControlMultiPoint) +export(setMBOControlNoisy) export(setMBOControlTermination) export(trafoLog) export(trafoSqrt) diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R index 3f79fb118..ba47d042e 100644 --- a/R/evalTargetFun.R +++ b/R/evalTargetFun.R @@ -25,13 +25,30 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { # short names and so on nevals = length(xs) ny = control$n.objectives + + # trafo X points + xs.trafo = lapply(xs, trafoValue, par = par.set) + + # handle noisy instances + if (isTRUE(control$noisy.instances > 1L)) { + nevals = nevals * control$noisy.instances + xs = rep(xs, each = control$noisy.instances) + extras = rep(extras, each = control$noisy.instances) + if (!control$noisy.self.replicating) { + xs.trafo = rep(xs.trafo, each = control$noisy.instances) + if (!is.na(control$noisy.instance.param)) { + inst.param = lapply(seq_len(control$noisy.instances), function(x) setNames(list(x), control$noisy.instance.param)) + xs.trafo = Map(c, xs.trafo, inst.param) + } + } + } + + num.format = control$output.num.format num.format.string = paste("%s = ", num.format, sep = "") dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer") imputeY = control$impute.y.fun - # trafo X points - xs.trafo = lapply(xs, trafoValue, par = par.set) # function to measure of fun call wrapFun = function(x) { @@ -43,6 +60,9 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { user.extras = attr(y, "extras") y = setAttribute(y, "extras", NULL) } + if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) { + user.extras = c(user.extras, x[control$noisy.instance.param]) + } st = proc.time() - st list(y = y, time = st[3], user.extras = user.extras) } @@ -56,6 +76,17 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", impute.error = if (is.null(imputeY)) NULL else identity) + # handle noisy instances of self.replicating functions + if (isTRUE(control$noisy.instances > 1L) && control$noisy.self.replicating) { + xs.trafo = rep(xs.trafo, each = control$noisy.instances) + res = lapply(res, function(r) { + lapply(seq_along(r$y), function(i) { + list(y = r$y[i], time = r$time / length(r$y), user.extras = c(r$user.extras, setNames(list(i), control$noisy.instance.param))) + }) + }) + res = unlist(res, recursive = FALSE) + } + # loop evals and to some post-processing for (i in seq_len(nevals)) { r = res[[i]]; x = xs[[i]]; x.trafo = xs.trafo[[i]]; dob = dobs[i] diff --git a/R/setMBOControlNoisy.R b/R/setMBOControlNoisy.R new file mode 100644 index 000000000..7bd80cbc0 --- /dev/null +++ b/R/setMBOControlNoisy.R @@ -0,0 +1,30 @@ +#' @title Set multipoint proposal options. +#' @description +#' Extends an MBO control object with options for multipoint proposal. +#' @template arg_control +#' @param instances [\code{integer(1)}]\cr +#' How many instances of one parameter will be calculated? +#' @param instance.param [\code{character(1)}]\cr +#' What is the name of the function param that defines the instance? +#' @param self.replicating [\code{logical(1)}]\cr +#' TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect. +#' @return [\code{\link{MBOControl}}]. +#' @family MBOControl +#' @export +setMBOControlNoisy = function(control, + instances = NULL, + instance.param = NULL, + self.replicating = NULL) { + + assertClass(control, "MBOControl") + + control$noisy.instances = assertInt(instances, lower = 1L, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.instances + control$noisy.self.replicating = assertFlag(self.replicating, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.self.replicating %??% FALSE + control$noisy.instance.param = assertString(instance.param, null.ok = TRUE, na.ok = TRUE) %??% control$noisy.instance.param %??% ifelse(control$noisy.self.replicating, "noisy.repl", NA_character_) + + if (control$noisy.self.replicating && control$noisy.instance.param != "noisy.repl") { + stop("You can not change the instance.param for self replicating functions.") + } + + return(control) +} diff --git a/man/makeMBOControl.Rd b/man/makeMBOControl.Rd index 55847a818..0519b71ea 100644 --- a/man/makeMBOControl.Rd +++ b/man/makeMBOControl.Rd @@ -120,5 +120,6 @@ Creates a control object for MBO optimization. Other MBOControl: \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } diff --git a/man/setMBOControlInfill.Rd b/man/setMBOControlInfill.Rd index b6ccc585a..184f45b62 100644 --- a/man/setMBOControlInfill.Rd +++ b/man/setMBOControlInfill.Rd @@ -156,5 +156,6 @@ minimizes that. Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlMultiObj}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } diff --git a/man/setMBOControlMultiObj.Rd b/man/setMBOControlMultiObj.Rd index 4dfba869b..e786ca244 100644 --- a/man/setMBOControlMultiObj.Rd +++ b/man/setMBOControlMultiObj.Rd @@ -119,5 +119,6 @@ et.al. (eds.), IEEE, 2005, ISBN 0-7803-9363-5, pp. 2138-2145 Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } diff --git a/man/setMBOControlMultiPoint.Rd b/man/setMBOControlMultiPoint.Rd index d939eb10f..7996d3b0d 100644 --- a/man/setMBOControlMultiPoint.Rd +++ b/man/setMBOControlMultiPoint.Rd @@ -88,5 +88,6 @@ Extends an MBO control object with options for multipoint proposal. Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, + \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } diff --git a/man/setMBOControlNoisy.Rd b/man/setMBOControlNoisy.Rd new file mode 100644 index 000000000..195fe1cf1 --- /dev/null +++ b/man/setMBOControlNoisy.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/setMBOControlNoisy.R +\name{setMBOControlNoisy} +\alias{setMBOControlNoisy} +\title{Set multipoint proposal options.} +\usage{ +setMBOControlNoisy(control, instances = NULL, instance.param = NULL) +} +\arguments{ +\item{control}{[\code{\link{MBOControl}}]\cr +Control object for mbo.} + +\item{instances}{[\code{integer(1)}]\cr +How many instances of one parameter will be calculated?} + +\item{instance.param}{[\code{character(1)}]\cr +What is the name of the function param that defines the instance?} +} +\value{ +[\code{\link{MBOControl}}]. +} +\description{ +Extends an MBO control object with options for multipoint proposal. +} +\seealso{ +Other MBOControl: \code{\link{makeMBOControl}}, + \code{\link{setMBOControlInfill}}, + \code{\link{setMBOControlMultiObj}}, + \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlTermination}} +} diff --git a/man/setMBOControlTermination.Rd b/man/setMBOControlTermination.Rd index 0446ec466..123b4cb09 100644 --- a/man/setMBOControlTermination.Rd +++ b/man/setMBOControlTermination.Rd @@ -77,5 +77,6 @@ print(res) Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlMultiObj}}, - \code{\link{setMBOControlMultiPoint}} + \code{\link{setMBOControlMultiPoint}}, + \code{\link{setMBOControlNoisy}} } diff --git a/tests/testthat/test_mbo_km.R b/tests/testthat/test_mbo_km.R index a8db0a811..a69e33f51 100644 --- a/tests/testthat/test_mbo_km.R +++ b/tests/testthat/test_mbo_km.R @@ -1,6 +1,6 @@ -context("mbo km") +context("mbo noisy") -test_that("mbo works with km", { +test_that("mbo works with multiple instances of noisy problems", { des = testd.fsphere.2d des$y = apply(des, 1, testf.fsphere.2d) learner = makeLearner("regr.km", nugget.estim = TRUE) diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R new file mode 100644 index 000000000..ac3c4fa1c --- /dev/null +++ b/tests/testthat/test_mbo_noisy.R @@ -0,0 +1,51 @@ +context("mbo noisy") + +test_that("mbo works with multiple instances of noisy problems", { + ps = makeNumericParamSet("x", 1, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) x^2 + rnorm(1, 0, 0.5), + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl() + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, instances = 5L) + or = mbo(fun, control = ctrl) + expect_true(all(table(opdf$x) == 5) +}) + +test_that("mbo works with multiple fixed instances of noisy problems", { + ps = makeNumericParamSet("x", 1, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) x$x^2 + rnorm(1, (x$i - 2)/100, 0.05), + par.set = ps, + noisy = TRUE, + has.simple.signature = FALSE + ) + ctrl = makeMBOControl() + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, instances = 5L, instance.param = "i") + or = mbo(fun, control = ctrl) + opdf = as.data.frame(or$opt.path) + expect_true(all(opdf$i %in% 1:5)) + expect_true(all(table(opdf$x) == 5) +}) + +test_that("mbo works with self replicating instances of noisy problems", { + ps = makeNumericParamSet("x", 1, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) x^2 + rnorm(5, 0.01), + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl() + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, instances = 5L, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + opdf = as.data.frame(or$opt.path) + expect_true(all(opdf$noisy.repl %in% 1:5)) + expect_true(all(table(opdf$x) == 5) +}) From 6f0a53746e0b53c1077e72585c72106ccd12a4e7 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Mon, 5 Feb 2018 17:22:57 +0100 Subject: [PATCH 2/8] fix tests --- man/setMBOControlNoisy.Rd | 6 +++++- tests/testthat/test_mbo_noisy.R | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/man/setMBOControlNoisy.Rd b/man/setMBOControlNoisy.Rd index 195fe1cf1..38d44610c 100644 --- a/man/setMBOControlNoisy.Rd +++ b/man/setMBOControlNoisy.Rd @@ -4,7 +4,8 @@ \alias{setMBOControlNoisy} \title{Set multipoint proposal options.} \usage{ -setMBOControlNoisy(control, instances = NULL, instance.param = NULL) +setMBOControlNoisy(control, instances = NULL, instance.param = NULL, + self.replicating = NULL) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr @@ -15,6 +16,9 @@ How many instances of one parameter will be calculated?} \item{instance.param}{[\code{character(1)}]\cr What is the name of the function param that defines the instance?} + +\item{self.replicating}{[\code{logical(1)}]\cr +TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect.} } \value{ [\code{\link{MBOControl}}]. diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R index ac3c4fa1c..9cb433858 100644 --- a/tests/testthat/test_mbo_noisy.R +++ b/tests/testthat/test_mbo_noisy.R @@ -12,7 +12,8 @@ test_that("mbo works with multiple instances of noisy problems", { ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) ctrl = setMBOControlNoisy(ctrl, instances = 5L) or = mbo(fun, control = ctrl) - expect_true(all(table(opdf$x) == 5) + opdf = as.data.frame(or$opt.path) + expect_true(all(table(opdf$x) == 5)) }) test_that("mbo works with multiple fixed instances of noisy problems", { @@ -30,7 +31,7 @@ test_that("mbo works with multiple fixed instances of noisy problems", { or = mbo(fun, control = ctrl) opdf = as.data.frame(or$opt.path) expect_true(all(opdf$i %in% 1:5)) - expect_true(all(table(opdf$x) == 5) + expect_true(all(table(opdf$x) == 5)) }) test_that("mbo works with self replicating instances of noisy problems", { @@ -47,5 +48,5 @@ test_that("mbo works with self replicating instances of noisy problems", { or = mbo(fun, control = ctrl) opdf = as.data.frame(or$opt.path) expect_true(all(opdf$noisy.repl %in% 1:5)) - expect_true(all(table(opdf$x) == 5) + expect_true(all(table(opdf$x) == 5)) }) From d8dc63189dab33a9e4720c1dfb63452302c88010 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Tue, 6 Feb 2018 14:22:15 +0100 Subject: [PATCH 3/8] error handling --- R/evalTargetFun.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R index ba47d042e..125fe33ee 100644 --- a/R/evalTargetFun.R +++ b/R/evalTargetFun.R @@ -80,9 +80,13 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { if (isTRUE(control$noisy.instances > 1L) && control$noisy.self.replicating) { xs.trafo = rep(xs.trafo, each = control$noisy.instances) res = lapply(res, function(r) { - lapply(seq_along(r$y), function(i) { - list(y = r$y[i], time = r$time / length(r$y), user.extras = c(r$user.extras, setNames(list(i), control$noisy.instance.param))) - }) + if (is.error(r)) { + rep(list(r), control$noisy.instances) + } else { + lapply(seq_along(r$y), function(i) { + list(y = r$y[i], time = r$time / length(r$y), user.extras = c(r$user.extras, setNames(list(i), control$noisy.instance.param))) + }) + } }) res = unlist(res, recursive = FALSE) } From 6e356063fa52d1611e8b770a1f8b617fc3b7240a Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Fri, 29 Jun 2018 15:41:11 +0200 Subject: [PATCH 4/8] smbo noisy fix --- R/SMBO.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/SMBO.R b/R/SMBO.R index 89caad62f..0839efaa5 100644 --- a/R/SMBO.R +++ b/R/SMBO.R @@ -49,6 +49,8 @@ initSMBO = function(par.set, design, learner = NULL, control, minimize = rep(TRU #' Outcome of the optimization. #' For multiple results use a list. #' For a result of a multi-objective function use a numeric vector. +#' For multiple results of for noisy instances use a list. +#' Each list element should correspond to one x value. #' #' @return [\code{\link{OptState}}] #' @export From db2239f7d6ed19ab7c981a3949161737250d539d Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Wed, 14 Nov 2018 16:03:30 +0100 Subject: [PATCH 5/8] make noisy repls flexible --- DESCRIPTION | 2 +- R/evalTargetFun.R | 88 ++++++++++++++++++--------------- R/setMBOControlNoisy.R | 18 ++----- man/makeMBOControl.Rd | 9 ++-- man/plotMBOResult.Rd | 8 +-- man/renderExampleRunPlot.Rd | 4 +- man/setMBOControlInfill.Rd | 11 +++-- man/setMBOControlMultiObj.Rd | 1 + man/setMBOControlMultiPoint.Rd | 8 +-- man/setMBOControlNoisy.Rd | 10 +--- man/setMBOControlTermination.Rd | 1 + man/updateSMBO.Rd | 4 +- tests/testthat/test_mbo_noisy.R | 30 +++++------ 13 files changed, 97 insertions(+), 97 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 55d0869ea..16e8a09b6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -61,5 +61,5 @@ LazyData: yes Encoding: UTF-8 ByteCompile: yes Version: 1.1.3 -RoxygenNote: 6.0.1 +RoxygenNote: 6.1.1 VignetteBuilder: knitr diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R index 125fe33ee..b41a7c0b6 100644 --- a/R/evalTargetFun.R +++ b/R/evalTargetFun.R @@ -25,60 +25,65 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { # short names and so on nevals = length(xs) ny = control$n.objectives + imputeY = control$impute.y.fun # trafo X points xs.trafo = lapply(xs, trafoValue, par = par.set) - # handle noisy instances - if (isTRUE(control$noisy.instances > 1L)) { - nevals = nevals * control$noisy.instances - xs = rep(xs, each = control$noisy.instances) - extras = rep(extras, each = control$noisy.instances) - if (!control$noisy.self.replicating) { - xs.trafo = rep(xs.trafo, each = control$noisy.instances) - if (!is.na(control$noisy.instance.param)) { - inst.param = lapply(seq_len(control$noisy.instances), function(x) setNames(list(x), control$noisy.instance.param)) - xs.trafo = Map(c, xs.trafo, inst.param) - } - } - } - - - num.format = control$output.num.format - num.format.string = paste("%s = ", num.format, sep = "") - dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer") - imputeY = control$impute.y.fun - - # function to measure of fun call - wrapFun = function(x) { - st = proc.time() - y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem))) - user.extras = list() + wrapFun = function(x) { + st = proc.time() + y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem))) + user.extras = list() # here we extract additional stuff which the user wants to log in the opt path - if (hasAttributes(y, "extras")) { - user.extras = attr(y, "extras") - y = setAttribute(y, "extras", NULL) - } - if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) { - user.extras = c(user.extras, x[control$noisy.instance.param]) - } - st = proc.time() - st - list(y = y, time = st[3], user.extras = user.extras) + if (hasAttributes(y, "extras")) { + user.extras = attr(y, "extras") + y = setAttribute(y, "extras", NULL) + } + if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) { + user.extras = c(user.extras, x[control$noisy.instance.param]) } + st = proc.time() - st + list(y = y, time = st[3], user.extras = user.extras) + } # do we have a valid y object? isYValid = function(y) { - !is.error(y) && testNumeric(y, len = ny, any.missing = FALSE, finite = TRUE) + if (!isTRUE(control$noisy.self.replicating)) { + len = NULL + } else { + len = ny + } + !is.error(y) && testNumeric(y, len = len, any.missing = FALSE, finite = TRUE) } # return error objects if we impute - res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", - impute.error = if (is.null(imputeY)) NULL else identity) + res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", impute.error = if (is.null(imputeY)) NULL else identity) + + # handle noisy instances + if (isTRUE(control$noisy.self.replicating)) { + nevals.each = lengths(extractSubList(res, "y", simplify = FALSE)) + nevals = sum(nevals.each) + + # replications for opt path stuff + repVec = function(x, fun = replicate) { + unlist(Map(fun, nevals.each, x, simplify = FALSE), recursive = FALSE) + } + xs = repVec(xs) + xs.trafo = repVec(xs.trafo) + + #set extras to NA that are only important for the first point + setNAfun = function(n, xs, ...) { + res = replicate(n = n, expr = xs, ...) + res[-1] = lapply(res[-1], function(x) { + x[c("train.time", "error.model", "propose.time")] = NA + x + }) + res + } + extras = repVec(extras, setNAfun) - # handle noisy instances of self.replicating functions - if (isTRUE(control$noisy.instances > 1L) && control$noisy.self.replicating) { - xs.trafo = rep(xs.trafo, each = control$noisy.instances) + # handle result list res = lapply(res, function(r) { if (is.error(r)) { rep(list(r), control$noisy.instances) @@ -91,6 +96,9 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { res = unlist(res, recursive = FALSE) } + num.format.string = paste("%s = ", control$output.num.format, sep = "") + dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer") + # loop evals and to some post-processing for (i in seq_len(nevals)) { r = res[[i]]; x = xs[[i]]; x.trafo = xs.trafo[[i]]; dob = dobs[i] diff --git a/R/setMBOControlNoisy.R b/R/setMBOControlNoisy.R index 7bd80cbc0..91383f81b 100644 --- a/R/setMBOControlNoisy.R +++ b/R/setMBOControlNoisy.R @@ -2,29 +2,17 @@ #' @description #' Extends an MBO control object with options for multipoint proposal. #' @template arg_control -#' @param instances [\code{integer(1)}]\cr -#' How many instances of one parameter will be calculated? -#' @param instance.param [\code{character(1)}]\cr -#' What is the name of the function param that defines the instance? #' @param self.replicating [\code{logical(1)}]\cr #' TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect. #' @return [\code{\link{MBOControl}}]. #' @family MBOControl #' @export -setMBOControlNoisy = function(control, - instances = NULL, - instance.param = NULL, - self.replicating = NULL) { +setMBOControlNoisy = function(control, self.replicating) { assertClass(control, "MBOControl") - control$noisy.instances = assertInt(instances, lower = 1L, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.instances - control$noisy.self.replicating = assertFlag(self.replicating, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.self.replicating %??% FALSE - control$noisy.instance.param = assertString(instance.param, null.ok = TRUE, na.ok = TRUE) %??% control$noisy.instance.param %??% ifelse(control$noisy.self.replicating, "noisy.repl", NA_character_) - - if (control$noisy.self.replicating && control$noisy.instance.param != "noisy.repl") { - stop("You can not change the instance.param for self replicating functions.") - } + control$noisy.self.replicating = assertFlag(self.replicating %??% control$noisy.self.replicating %??% TRUE, na.ok = FALSE) + control$noisy.instance.param = "noisy.repl" return(control) } diff --git a/man/makeMBOControl.Rd b/man/makeMBOControl.Rd index 0519b71ea..a93f06d45 100644 --- a/man/makeMBOControl.Rd +++ b/man/makeMBOControl.Rd @@ -7,10 +7,10 @@ \usage{ makeMBOControl(n.objectives = 1L, propose.points = 1L, final.method = "best.true.y", final.evals = 0L, y.name = "y", - impute.y.fun = NULL, trafo.y.fun = NULL, suppress.eval.errors = TRUE, - save.on.disk.at = integer(0L), save.on.disk.at.time = Inf, - save.file.path = file.path(getwd(), "mlr_run.RData"), - store.model.at = NULL, resample.at = integer(0), + impute.y.fun = NULL, trafo.y.fun = NULL, + suppress.eval.errors = TRUE, save.on.disk.at = integer(0L), + save.on.disk.at.time = Inf, save.file.path = file.path(getwd(), + "mlr_run.RData"), store.model.at = NULL, resample.at = integer(0), resample.desc = makeResampleDesc("CV", iter = 10), resample.measures = list(mse), output.num.format = "\%.3g", on.surrogate.error = "stop") @@ -123,3 +123,4 @@ Other MBOControl: \code{\link{setMBOControlInfill}}, \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/plotMBOResult.Rd b/man/plotMBOResult.Rd index 54d88ea4b..30c679486 100644 --- a/man/plotMBOResult.Rd +++ b/man/plotMBOResult.Rd @@ -6,11 +6,11 @@ \alias{plot.MBOMultiObjResult} \title{MBO Result Plotting} \usage{ -\method{plot}{MBOSingleObjResult}(x, iters = NULL, pause = interactive(), - ...) +\method{plot}{MBOSingleObjResult}(x, iters = NULL, + pause = interactive(), ...) -\method{plot}{MBOMultiObjResult}(x, iters = NULL, pause = interactive(), - ...) +\method{plot}{MBOMultiObjResult}(x, iters = NULL, + pause = interactive(), ...) } \arguments{ \item{x}{[\code{MBOResult}]\cr diff --git a/man/renderExampleRunPlot.Rd b/man/renderExampleRunPlot.Rd index de888f594..c368bf75f 100644 --- a/man/renderExampleRunPlot.Rd +++ b/man/renderExampleRunPlot.Rd @@ -7,8 +7,8 @@ exampleRunMultiObj objects.} \usage{ renderExampleRunPlot(object, iter, densregion = TRUE, se.factor = 1, single.prop.point.plots = FALSE, xlim = NULL, ylim = NULL, - point.size = 3, line.size = 1, trafo = NULL, colors = c("red", "blue", - "green"), ...) + point.size = 3, line.size = 1, trafo = NULL, colors = c("red", + "blue", "green"), ...) } \arguments{ \item{object}{[\code{function}]\cr diff --git a/man/setMBOControlInfill.Rd b/man/setMBOControlInfill.Rd index 184f45b62..68f911a83 100644 --- a/man/setMBOControlInfill.Rd +++ b/man/setMBOControlInfill.Rd @@ -4,15 +4,17 @@ \alias{setMBOControlInfill} \title{Extends mbo control object with infill criteria and infill optimizer options.} \usage{ -setMBOControlInfill(control, crit = NULL, interleave.random.points = 0L, - filter.proposed.points = NULL, filter.proposed.points.tol = NULL, - opt = "focussearch", opt.restarts = NULL, opt.focussearch.maxit = NULL, +setMBOControlInfill(control, crit = NULL, + interleave.random.points = 0L, filter.proposed.points = NULL, + filter.proposed.points.tol = NULL, opt = "focussearch", + opt.restarts = NULL, opt.focussearch.maxit = NULL, opt.focussearch.points = NULL, opt.cmaes.control = NULL, opt.ea.maxit = NULL, opt.ea.mu = NULL, opt.ea.sbx.eta = NULL, opt.ea.sbx.p = NULL, opt.ea.pm.eta = NULL, opt.ea.pm.p = NULL, opt.ea.lambda = NULL, opt.nsga2.popsize = NULL, opt.nsga2.generations = NULL, opt.nsga2.cprob = NULL, - opt.nsga2.cdist = NULL, opt.nsga2.mprob = NULL, opt.nsga2.mdist = NULL) + opt.nsga2.cdist = NULL, opt.nsga2.mprob = NULL, + opt.nsga2.mdist = NULL) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr @@ -159,3 +161,4 @@ Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlMultiObj.Rd b/man/setMBOControlMultiObj.Rd index e786ca244..e13ceaec9 100644 --- a/man/setMBOControlMultiObj.Rd +++ b/man/setMBOControlMultiObj.Rd @@ -122,3 +122,4 @@ Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlMultiPoint.Rd b/man/setMBOControlMultiPoint.Rd index 7996d3b0d..232dffbeb 100644 --- a/man/setMBOControlMultiPoint.Rd +++ b/man/setMBOControlMultiPoint.Rd @@ -5,9 +5,10 @@ \title{Set multipoint proposal options.} \usage{ setMBOControlMultiPoint(control, method = NULL, cl.lie = NULL, - moimbo.objective = NULL, moimbo.dist = NULL, moimbo.selection = NULL, - moimbo.maxit = NULL, moimbo.sbx.eta = NULL, moimbo.sbx.p = NULL, - moimbo.pm.eta = NULL, moimbo.pm.p = NULL) + moimbo.objective = NULL, moimbo.dist = NULL, + moimbo.selection = NULL, moimbo.maxit = NULL, + moimbo.sbx.eta = NULL, moimbo.sbx.p = NULL, moimbo.pm.eta = NULL, + moimbo.pm.p = NULL) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr @@ -91,3 +92,4 @@ Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlNoisy}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlNoisy.Rd b/man/setMBOControlNoisy.Rd index 38d44610c..1e22eed84 100644 --- a/man/setMBOControlNoisy.Rd +++ b/man/setMBOControlNoisy.Rd @@ -4,19 +4,12 @@ \alias{setMBOControlNoisy} \title{Set multipoint proposal options.} \usage{ -setMBOControlNoisy(control, instances = NULL, instance.param = NULL, - self.replicating = NULL) +setMBOControlNoisy(control, self.replicating) } \arguments{ \item{control}{[\code{\link{MBOControl}}]\cr Control object for mbo.} -\item{instances}{[\code{integer(1)}]\cr -How many instances of one parameter will be calculated?} - -\item{instance.param}{[\code{character(1)}]\cr -What is the name of the function param that defines the instance?} - \item{self.replicating}{[\code{logical(1)}]\cr TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect.} } @@ -33,3 +26,4 @@ Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlMultiPoint}}, \code{\link{setMBOControlTermination}} } +\concept{MBOControl} diff --git a/man/setMBOControlTermination.Rd b/man/setMBOControlTermination.Rd index 9179b3ca1..b9c1ddb39 100644 --- a/man/setMBOControlTermination.Rd +++ b/man/setMBOControlTermination.Rd @@ -86,3 +86,4 @@ Other MBOControl: \code{\link{makeMBOControl}}, \code{\link{setMBOControlMultiPoint}}, \code{\link{setMBOControlNoisy}} } +\concept{MBOControl} diff --git a/man/updateSMBO.Rd b/man/updateSMBO.Rd index 31e2f4313..420166083 100644 --- a/man/updateSMBO.Rd +++ b/man/updateSMBO.Rd @@ -18,7 +18,9 @@ One row per set of x values.} \item{y}{[\code{numeric|list}] Outcome of the optimization. For multiple results use a list. -For a result of a multi-objective function use a numeric vector.} +For a result of a multi-objective function use a numeric vector. +For multiple results of for noisy instances use a list. +Each list element should correspond to one x value.} } \value{ [\code{\link{OptState}}] diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R index 9cb433858..d662d2af6 100644 --- a/tests/testthat/test_mbo_noisy.R +++ b/tests/testthat/test_mbo_noisy.R @@ -16,37 +16,37 @@ test_that("mbo works with multiple instances of noisy problems", { expect_true(all(table(opdf$x) == 5)) }) -test_that("mbo works with multiple fixed instances of noisy problems", { +test_that("mbo works with self replicating instances of noisy problems", { ps = makeNumericParamSet("x", 1, -7, 7) fun = smoof::makeSingleObjectiveFunction( - fn = function(x) x$x^2 + rnorm(1, (x$i - 2)/100, 0.05), + fn = function(x) x^2 + rnorm(5, 0.01), par.set = ps, - noisy = TRUE, - has.simple.signature = FALSE + noisy = TRUE ) ctrl = makeMBOControl() ctrl = setMBOControlTermination(ctrl, iters = 5L) ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) - ctrl = setMBOControlNoisy(ctrl, instances = 5L, instance.param = "i") + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) or = mbo(fun, control = ctrl) opdf = as.data.frame(or$opt.path) - expect_true(all(opdf$i %in% 1:5)) + expect_true(all(opdf$noisy.repl %in% 1:5)) expect_true(all(table(opdf$x) == 5)) -}) -test_that("mbo works with self replicating instances of noisy problems", { - ps = makeNumericParamSet("x", 1, -7, 7) + # now the function has varying n results + i = 0 fun = smoof::makeSingleObjectiveFunction( - fn = function(x) x^2 + rnorm(5, 0.01), + fn = function(x) { + i <<- i + 1 + x^2 + rnorm(i, 0.01) + }, par.set = ps, noisy = TRUE ) - ctrl = makeMBOControl() - ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlTermination(ctrl, iters = 2L) ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) - ctrl = setMBOControlNoisy(ctrl, instances = 5L, self.replicating = TRUE) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) or = mbo(fun, control = ctrl) opdf = as.data.frame(or$opt.path) - expect_true(all(opdf$noisy.repl %in% 1:5)) - expect_true(all(table(opdf$x) == 5)) + expect_true(all(opdf$noisy.repl == unlist(lapply(1:6, function(x) head(1:6,x))))) }) + From 5a58dccfdf9e9e81467da87f4506fcde6502b079 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Wed, 14 Nov 2018 16:10:56 +0100 Subject: [PATCH 6/8] fix tests --- R/evalTargetFun.R | 2 +- tests/testthat/test_mbo_noisy.R | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R index b41a7c0b6..d9deb9d59 100644 --- a/R/evalTargetFun.R +++ b/R/evalTargetFun.R @@ -135,7 +135,7 @@ evalTargetFun.OptState = function(opt.state, xs, extras) { # showInfo - use the trafo'd value here! showInfo(getOptProblemShowInfo(opt.problem), "[mbo] %i: %s : %s : %.1f secs%s : %s", dob, - paramValueToString(par.set, x.trafo, num.format = num.format), + paramValueToString(par.set, x.trafo, num.format = control$output.num.format), collapse(sprintf(num.format.string, control$y.name, y2), ", "), ytime, ifelse(y.valid, "", " (imputed)"), diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R index d662d2af6..ae2a567e6 100644 --- a/tests/testthat/test_mbo_noisy.R +++ b/tests/testthat/test_mbo_noisy.R @@ -1,21 +1,5 @@ context("mbo noisy") -test_that("mbo works with multiple instances of noisy problems", { - ps = makeNumericParamSet("x", 1, -7, 7) - fun = smoof::makeSingleObjectiveFunction( - fn = function(x) x^2 + rnorm(1, 0, 0.5), - par.set = ps, - noisy = TRUE - ) - ctrl = makeMBOControl() - ctrl = setMBOControlTermination(ctrl, iters = 5L) - ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) - ctrl = setMBOControlNoisy(ctrl, instances = 5L) - or = mbo(fun, control = ctrl) - opdf = as.data.frame(or$opt.path) - expect_true(all(table(opdf$x) == 5)) -}) - test_that("mbo works with self replicating instances of noisy problems", { ps = makeNumericParamSet("x", 1, -7, 7) fun = smoof::makeSingleObjectiveFunction( From 51a6a0af3b55a4143614012171bcbffc064813a6 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Fri, 16 Nov 2018 13:02:53 +0100 Subject: [PATCH 7/8] fix test? --- tests/testthat/test_mbo_noisy.R | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R index ae2a567e6..8bbe50954 100644 --- a/tests/testthat/test_mbo_noisy.R +++ b/tests/testthat/test_mbo_noisy.R @@ -26,11 +26,31 @@ test_that("mbo works with self replicating instances of noisy problems", { par.set = ps, noisy = TRUE ) + ctrl = makeMBOControl() ctrl = setMBOControlTermination(ctrl, iters = 2L) ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) or = mbo(fun, control = ctrl) opdf = as.data.frame(or$opt.path) expect_true(all(opdf$noisy.repl == unlist(lapply(1:6, function(x) head(1:6,x))))) + + # returns the right result + ps = makeNumericParamSet("x", 2, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) { + x = sum(unlist(x)) + res = x^2 + rnorm(5, 0.01) + if (abs(x)>5) res[1] = -10 + return(res) + }, + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl(final.method = "best.predicted") + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + expect_true(abs(sum(or$x$x))<5) }) From 4f0dac33f61911e5bb7e67b2e58fcb4ce02b6aa1 Mon Sep 17 00:00:00 2001 From: Jakob Richter Date: Thu, 14 Mar 2019 12:00:46 +0100 Subject: [PATCH 8/8] maybe test errors in noisy --- tests/testthat/test_mbo_noisy.R | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R index 8bbe50954..85cca7981 100644 --- a/tests/testthat/test_mbo_noisy.R +++ b/tests/testthat/test_mbo_noisy.R @@ -52,5 +52,26 @@ test_that("mbo works with self replicating instances of noisy problems", { ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) or = mbo(fun, control = ctrl) expect_true(abs(sum(or$x$x))<5) + + # deals with errors in noisy repls + if (FALSE) { + ps = makeNumericParamSet("x", 2, -7, 7) + fun = smoof::makeSingleObjectiveFunction( + fn = function(x) { + x = sum(unlist(x)) + res = x^2 + rnorm(5, 0.01) + res = as.list(res) + res[[1]] = base::simpleError("Something went wrong!") + }, + par.set = ps, + noisy = TRUE + ) + ctrl = makeMBOControl(final.method = "best.predicted", impute.y.fun = function(x, y, opt.path) 0, on.surrogate.error = "warn") + ctrl = setMBOControlTermination(ctrl, iters = 5L) + ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L) + ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE) + or = mbo(fun, control = ctrl) + as.data.frame(or$opt.path) + } })