From 388846b87b3233018fb0582fd7e6b498b952da6c Mon Sep 17 00:00:00 2001 From: wlandau Date: Mon, 22 May 2023 12:10:42 -0400 Subject: [PATCH] collect arg of pop() --- NEWS.md | 1 + R/crew_controller.R | 22 ++++++++++++++++------ R/crew_controller_group.R | 17 +++++++++++++++-- man/crew_class_controller.Rd | 15 +++++++++++++-- man/crew_class_controller_group.Rd | 6 ++++++ 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/NEWS.md b/NEWS.md index c22e6920..fa51ccf4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,6 +13,7 @@ * Throttle auto-scaling with interval `self$router$seconds_interval` (#76). * Remove `clean()` and instead terminate lost workers on launch. * Fix examples. +* Add a `collect` argument to `pop()`. # crew 0.1.1 diff --git a/R/crew_controller.R b/R/crew_controller.R index c60dd58e..005e5f14 100644 --- a/R/crew_controller.R +++ b/R/crew_controller.R @@ -401,7 +401,8 @@ crew_class_controller <- R6::R6Class( #' value is a one-row data frame with the results, warnings, and errors. #' Otherwise, if there are no results available to collect, #' the return value is `NULL`. - #' @param scale Logical, whether to automatically call `scale_later()` + #' @param scale Logical of length 1, + #' whether to automatically call `scale()` #' to auto-scale workers to meet the demand of the task load. #' Auto-scaling might not actually happen if `throttle` is `TRUE`. #' If `TRUE`, then `collect()` runs first @@ -409,6 +410,10 @@ crew_class_controller <- R6::R6Class( #' of workers is not too high. Scaling up on `pop()` may be important #' for transient or nearly transient workers that tend to drop off #' quickly after doing little work. + #' @param collect Logical of length 1. If `scale` is `FALSE`, + #' whether to call `collect()` + #' to pick up the results of completed tasks. This task collection + #' step always happens (with throttling) when `scale` is `TRUE`. #' @param throttle Whether to defer auto-scaling and task collection #' until the next request at least #' `self$router$seconds_interval` seconds from the original request. @@ -417,12 +422,17 @@ crew_class_controller <- R6::R6Class( #' and efficiency. #' @param controllers Not used. Included to ensure the signature is #' compatible with the analogous method of controller groups. - pop = function(scale = TRUE, throttle = TRUE, controllers = NULL) { - if_any( - scale, - self$scale(throttle = throttle), + pop = function( + scale = TRUE, + collect = TRUE, + throttle = TRUE, + controllers = NULL + ) { + if (scale) { + self$scale(throttle = throttle) + } else if (collect) { self$collect(throttle = throttle) - ) + } out <- NULL if (length(self$results) > 0L) { task <- self$results[[1L]] diff --git a/R/crew_controller_group.R b/R/crew_controller_group.R index 7811172f..aabe5b74 100644 --- a/R/crew_controller_group.R +++ b/R/crew_controller_group.R @@ -294,6 +294,10 @@ crew_class_controller_group <- R6::R6Class( #' of workers is not too high. Scaling up on `pop()` may be important #' for transient or nearly transient workers that tend to drop off #' quickly after doing little work. + #' @param collect Logical of length 1. If `scale` is `FALSE`, + #' whether to call `collect()` + #' to pick up the results of completed tasks. This task collection + #' step always happens (with throttling) when `scale` is `TRUE`. #' @param throttle If `scale` is `TRUE`, whether to defer auto-scaling #' until the next auto-scaling request at least #' `self$router$seconds_interval` seconds from the original request. @@ -302,10 +306,19 @@ crew_class_controller_group <- R6::R6Class( #' and efficiency. #' @param controllers Character vector of controller names. #' Set to `NULL` to select all controllers. - pop = function(scale = TRUE, throttle = TRUE, controllers = NULL) { + pop = function( + scale = TRUE, + collect = TRUE, + throttle = TRUE, + controllers = NULL + ) { control <- private$select_controllers(controllers) for (controller in control) { - out <- controller$pop(scale = scale, throttle = throttle) + out <- controller$pop( + scale = scale, + collect = collect, + throttle = throttle + ) if (!is.null(out)) { return(out) } diff --git a/man/crew_class_controller.Rd b/man/crew_class_controller.Rd index 708259df..9af5e995 100644 --- a/man/crew_class_controller.Rd +++ b/man/crew_class_controller.Rd @@ -421,13 +421,19 @@ list as applicable and moves them to the \code{results} list. \subsection{Method \code{pop()}}{ Pop a completed task from the results data frame. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{crew_class_controller$pop(scale = TRUE, throttle = TRUE, controllers = NULL)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{crew_class_controller$pop( + scale = TRUE, + collect = TRUE, + throttle = TRUE, + controllers = NULL +)}\if{html}{\out{
}} } \subsection{Arguments}{ \if{html}{\out{
}} \describe{ -\item{\code{scale}}{Logical, whether to automatically call \code{scale_later()} +\item{\code{scale}}{Logical of length 1, +whether to automatically call \code{scale()} to auto-scale workers to meet the demand of the task load. Auto-scaling might not actually happen if \code{throttle} is \code{TRUE}. If \code{TRUE}, then \code{collect()} runs first @@ -436,6 +442,11 @@ of workers is not too high. Scaling up on \code{pop()} may be important for transient or nearly transient workers that tend to drop off quickly after doing little work.} +\item{\code{collect}}{Logical of length 1. If \code{scale} is \code{FALSE}, +whether to call \code{collect()} +to pick up the results of completed tasks. This task collection +step always happens (with throttling) when \code{scale} is \code{TRUE}.} + \item{\code{throttle}}{Whether to defer auto-scaling and task collection until the next request at least \code{self$router$seconds_interval} seconds from the original request. diff --git a/man/crew_class_controller_group.Rd b/man/crew_class_controller_group.Rd index 15729736..eb10fe2e 100644 --- a/man/crew_class_controller_group.Rd +++ b/man/crew_class_controller_group.Rd @@ -400,6 +400,7 @@ Pop a completed task from the results data frame. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{crew_class_controller_group$pop( scale = TRUE, + collect = TRUE, throttle = TRUE, controllers = NULL )}\if{html}{\out{
}} @@ -416,6 +417,11 @@ of workers is not too high. Scaling up on \code{pop()} may be important for transient or nearly transient workers that tend to drop off quickly after doing little work.} +\item{\code{collect}}{Logical of length 1. If \code{scale} is \code{FALSE}, +whether to call \code{collect()} +to pick up the results of completed tasks. This task collection +step always happens (with throttling) when \code{scale} is \code{TRUE}.} + \item{\code{throttle}}{If \code{scale} is \code{TRUE}, whether to defer auto-scaling until the next auto-scaling request at least \code{self$router$seconds_interval} seconds from the original request.