...

R/CallbackEvaluation.R

@@ -34,15 +34,22 @@ CallbackEvaluation= R6Class("CallbackEvaluation",
 #'
 #' @description
 #' Function to create a [CallbackEvaluation].
+#' Predefined callbacks are stored in the [dictionary][mlr3misc::Dictionary] [mlr_callbacks] and can be retrieved with [clbk()].
+#'
+#' Evaluation callbacks are called at different stages of the resampling process.
+#' The stages are prefixed with `on_*`.
 #'
 #' ```
-#' Start Evaluation on Worker
+#' Start Resampling Iteration on Worker
 #'  - on_evaluation_begin
 #'  - on_evaluation_before_train
 #'  - on_evaluation_before_predict
 #'  - on_evaluation_end
-#' End Evaluation on Worker
+#' End Resampling Iteration on Worker
 #' ```
+#'
+#' See also the section on parameters for more information on the stages.
+#' A evaluation callback works with [ContextEvaluation].
 #
 #' @details
 #' When implementing a callback, each function must have two arguments named `callback` and `context`.
@@ -84,15 +91,15 @@ callback_evaluation = function(
     on_evaluation_begin,
     on_evaluation_before_train,
     on_evaluation_before_predict,
-    on_evaluation_end ),
+    on_evaluation_end),
     c(
       "on_evaluation_begin",
       "on_evaluation_before_train",
       "on_evaluation_before_predict",
       "on_evaluation_end"
     )), is.null)
 
-  walk(stages, function(stage) assert_function(stage, args = c("callback", "context")))
+  stages = map(stages, function(stage) crate(assert_function(stage, args = c("callback", "context"))))
   callback = CallbackEvaluation$new(id, label, man)
   iwalk(stages, function(stage, name) callback[[name]] = stage)
   callback

R/ContextEvaluation.R

@@ -25,30 +25,10 @@ ContextEvaluation = R6Class("ContextEvaluation",
     #' The resampling is unchanged during the evaluation.
     resampling = NULL,
 
-    #' @field param_values `list()`\cr
-    #' The parameter values to be used.
-    #' Is usually only set while tuning.
-    param_values = NULL,
-
     #' @field iteration (`integer()`)\cr
     #' The current iteration.
     iteration = NULL,
 
-    #' @field sets (`list()`)\cr
-    #' The train and test set.
-    #' The sets are available on stage `on_evaluation_before_train``.
-    sets = NULL,
-
-    #' @field test_set (`integer()`)\cr
-    #' Validation test set.
-    #' The set is only available when using internal validation.
-    test_set = NULL,
-
-    #' @field predict_sets (`list()`)\cr
-    #' The prediction sets stored in `learner$predict_sets`.
-    #' The sets are available on stage `on_evaluation_before_predict`.
-    predict_sets = NULL,
-
     #' @field pdatas (List of [PredictionData])\cr
     #' The prediction data.
     #' The data is available on stage `on_evaluation_end`.
@@ -68,9 +48,9 @@ ContextEvaluation = R6Class("ContextEvaluation",
     #' The learner to be evaluated.
     #' @param resampling ([Resampling])\cr
     #' The resampling strategy to be used.
-    #' @param param_values (`list()`)\cr
-    #' The parameter values to be used.
-    initialize = function(task, learner, resampling, param_values, iteration) {
+    #' @param iteration (`integer()`)\cr
+    #' The current iteration.
+    initialize = function(task, learner, resampling, iteration) {
       # no assertions to avoid overhead
       self$task = task
       self$learner = learner

R/worker.R

@@ -265,7 +265,7 @@ workhorse = function(
   unmarshal = TRUE,
   callbacks = NULL
   ) {
-  ctx = ContextEvaluation$new(task, learner, resampling, param_values, iteration)
+  ctx = ContextEvaluation$new(task, learner, resampling, iteration)
 
   call_back("on_evaluation_begin", callbacks, ctx)
 
@@ -308,7 +308,7 @@ workhorse = function(
   lg$info("%s learner '%s' on task '%s' (iter %i/%i)",
     if (mode == "train") "Applying" else "Hotstarting", learner$id, task$id, iteration, resampling$iters)
 
-  ctx$sets = list(
+  sets = list(
     train = resampling$train_set(iteration),
     test = resampling$test_set(iteration)
   )
@@ -323,11 +323,11 @@ workhorse = function(
 
   validate = get0("validate", learner)
 
-  ctx$test_set = if (identical(validate, "test")) ctx$sets$test
+  ctx$test_set = if (identical(validate, "test")) sets$test
 
   call_back("on_evaluation_before_train", callbacks, ctx)
 
-  train_result = learner_train(learner, task, ctx$sets[["train"]], ctx$test_set, mode = mode)
+  train_result = learner_train(learner, task, sets[["train"]], ctx$test_set, mode = mode)
   ctx$learner = learner = train_result$learner
 
   # process the model so it can be used for prediction (e.g. marshal for callr prediction), but also
@@ -338,20 +338,20 @@ workhorse = function(
   )
 
   # predict for each set
-  ctx$predict_sets = learner$predict_sets
+  predict_sets = learner$predict_sets
 
   # creates the tasks and row_ids for all selected predict sets
-  pred_data = prediction_tasks_and_sets(task, train_result, validate, ctx$sets, ctx$predict_sets)
+  pred_data = prediction_tasks_and_sets(task, train_result, validate, sets, predict_sets)
 
   call_back("on_evaluation_before_predict", callbacks, ctx)
 
   pdatas = Map(function(set, row_ids, task) {
     lg$debug("Creating Prediction for predict set '%s'", set)
 
     learner_predict(learner, task, row_ids)
-  }, set = ctx$predict_sets, row_ids = pred_data$sets, task = pred_data$tasks)
+  }, set = predict_sets, row_ids = pred_data$sets, task = pred_data$tasks)
 
-  if (!length(ctx$predict_sets)) {
+  if (!length(predict_sets)) {
     learner$state$predict_time = 0L
   }
   ctx$pdatas = discard(pdatas, is.null)