diff --git a/R/crew_client.R b/R/crew_client.R index 72f50120..049ce5a5 100644 --- a/R/crew_client.R +++ b/R/crew_client.R @@ -394,6 +394,19 @@ crew_class_client <- R6::R6Class( complete = as.integer(daemons[, "complete"]), socket = as.character(rownames(daemons)) ) + }, + #' @description Get the process IDs of the local process and the + #' `mirai` dispatcher (if started). + #' @return An integer vector of process IDs of the local process and the + #' `mirai` dispatcher (if started). + pids = function() { + out <- c(local = Sys.getpid()) + dispatcher <- private$.dispatcher + if (!is.null(dispatcher)) { + out <- c(out, ps::ps_pid(dispatcher)) + names(out)[2L] <- paste0("dispatcher-", private$.name) + } + out } ) ) diff --git a/R/crew_controller.R b/R/crew_controller.R index 5d19caba..d16fd95f 100644 --- a/R/crew_controller.R +++ b/R/crew_controller.R @@ -1386,6 +1386,15 @@ crew_class_controller <- R6::R6Class( mirai::stop_mirai(tasks) invisible() }, + #' @description Get the process IDs of the local process and the + #' `mirai` dispatcher (if started). + #' @return An integer vector of process IDs of the local process and the + #' `mirai` dispatcher (if started). + #' @param controllers Not used. Included to ensure the signature is + #' compatible with the analogous method of controller groups. + pids = function(controllers = NULL) { + private$.client$pids() + }, #' @description Terminate the workers and the `mirai` client. #' @return `NULL` (invisibly). #' @param controllers Not used. Included to ensure the signature is diff --git a/R/crew_controller_group.R b/R/crew_controller_group.R index 04a216f0..294b1f24 100644 --- a/R/crew_controller_group.R +++ b/R/crew_controller_group.R @@ -956,6 +956,18 @@ crew_class_controller_group <- R6::R6Class( } tibble::as_tibble(do.call(what = rbind, args = out)) }, + #' @description Get the process IDs of the local process and the + #' `mirai` dispatchers (if started). + #' @return An integer vector of process IDs of the local process and the + #' `mirai` dispatcher (if started). + #' @param controllers Character vector of controller names. + #' Set to `NULL` to select all controllers. + pids = function(controllers = NULL) { + control <- private$.select_controllers(controllers) + out <- map(control, ~.x$pids()) + out <- map(unname(out), ~.x[names(.x) != "local"]) + c(local = Sys.getpid(), unlist(out)) + }, #' @description Terminate the workers and disconnect the client #' for one or more controllers. #' @return `NULL` (invisibly). diff --git a/R/crew_options_local.R b/R/crew_options_local.R index 7a03aec5..bcb3fde4 100644 --- a/R/crew_options_local.R +++ b/R/crew_options_local.R @@ -16,7 +16,7 @@ #' will go to different log files with informative suffixes. #' @examples #' crew_options_local() -crew_options_local = function( +crew_options_local <- function( log_directory = NULL, log_join = TRUE ) { diff --git a/R/crew_options_metrics.R b/R/crew_options_metrics.R index 480583b0..c9cbb93a 100644 --- a/R/crew_options_metrics.R +++ b/R/crew_options_metrics.R @@ -35,7 +35,7 @@ #' log entries written to `path`. #' @examples #' crew_options_metrics(path = "/dev/stdout") -crew_options_metrics = function(path = NULL, seconds_interval = 5) { +crew_options_metrics <- function(path = NULL, seconds_interval = 5) { out <- structure( list(path = path, seconds_interval = seconds_interval), class = c("crew_options_metrics", "crew_options") diff --git a/man/crew_class_client.Rd b/man/crew_class_client.Rd index 11d53401..438b35d5 100644 --- a/man/crew_class_client.Rd +++ b/man/crew_class_client.Rd @@ -73,6 +73,7 @@ condition variable.} \item \href{#method-crew_class_client-condition}{\code{crew_class_client$condition()}} \item \href{#method-crew_class_client-resolved}{\code{crew_class_client$resolved()}} \item \href{#method-crew_class_client-summary}{\code{crew_class_client$summary()}} +\item \href{#method-crew_class_client-pids}{\code{crew_class_client$pids()}} } } \if{html}{\out{
}} @@ -230,4 +231,19 @@ URL path periodically as a safeguard while managing workers. } } } +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-crew_class_client-pids}{}}} +\subsection{Method \code{pids()}}{ +Get the process IDs of the local process and the +\code{mirai} dispatcher (if started). +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{crew_class_client$pids()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +An integer vector of process IDs of the local process and the +\code{mirai} dispatcher (if started). +} +} } diff --git a/man/crew_class_controller.Rd b/man/crew_class_controller.Rd index 87e063a4..efa52011 100644 --- a/man/crew_class_controller.Rd +++ b/man/crew_class_controller.Rd @@ -93,6 +93,7 @@ auto-scaling is currently running} \item \href{#method-crew_class_controller-pop_backlog}{\code{crew_class_controller$pop_backlog()}} \item \href{#method-crew_class_controller-summary}{\code{crew_class_controller$summary()}} \item \href{#method-crew_class_controller-cancel}{\code{crew_class_controller$cancel()}} +\item \href{#method-crew_class_controller-pids}{\code{crew_class_controller$pids()}} \item \href{#method-crew_class_controller-terminate}{\code{crew_class_controller$terminate()}} } } @@ -1252,6 +1253,29 @@ Those names must have been manually supplied by \code{push()}.} } } \if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-crew_class_controller-pids}{}}} +\subsection{Method \code{pids()}}{ +Get the process IDs of the local process and the +\code{mirai} dispatcher (if started). +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{crew_class_controller$pids(controllers = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{controllers}}{Not used. Included to ensure the signature is +compatible with the analogous method of controller groups.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +An integer vector of process IDs of the local process and the +\code{mirai} dispatcher (if started). +} +} +\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-crew_class_controller-terminate}{}}} \subsection{Method \code{terminate()}}{ diff --git a/man/crew_class_controller_group.Rd b/man/crew_class_controller_group.Rd index 1d9c7a93..22465a5e 100644 --- a/man/crew_class_controller_group.Rd +++ b/man/crew_class_controller_group.Rd @@ -84,6 +84,7 @@ condition variable.} \item \href{#method-crew_class_controller_group-push_backlog}{\code{crew_class_controller_group$push_backlog()}} \item \href{#method-crew_class_controller_group-pop_backlog}{\code{crew_class_controller_group$pop_backlog()}} \item \href{#method-crew_class_controller_group-summary}{\code{crew_class_controller_group$summary()}} +\item \href{#method-crew_class_controller_group-pids}{\code{crew_class_controller_group$pids()}} \item \href{#method-crew_class_controller_group-terminate}{\code{crew_class_controller_group$terminate()}} } } @@ -1147,6 +1148,29 @@ class for specific information about the columns in the output. } } \if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-crew_class_controller_group-pids}{}}} +\subsection{Method \code{pids()}}{ +Get the process IDs of the local process and the +\code{mirai} dispatchers (if started). +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{crew_class_controller_group$pids(controllers = NULL)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{controllers}}{Character vector of controller names. +Set to \code{NULL} to select all controllers.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +An integer vector of process IDs of the local process and the +\code{mirai} dispatcher (if started). +} +} +\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-crew_class_controller_group-terminate}{}}} \subsection{Method \code{terminate()}}{ diff --git a/tests/testthat/test-crew_controller_group.R b/tests/testthat/test-crew_controller_group.R index 125c32ac..669e4e9a 100644 --- a/tests/testthat/test-crew_controller_group.R +++ b/tests/testthat/test-crew_controller_group.R @@ -39,12 +39,14 @@ crew_test("crew_controller_group()", { expect_null(x$controllers[[index]]$client$started) } expect_false(x$started()) + expect_equal(length(x$pids()), 1L) x$start() expect_true(x$started()) expect_true(x$empty()) expect_false(x$saturated()) expect_true(x$empty(controllers = "a")) expect_true(x$empty(controllers = "b")) + expect_equal(length(x$pids()), 3L) for (index in seq_len(2)) { expect_true(x$controllers[[index]]$client$started) } diff --git a/tests/testthat/test-crew_controller_local.R b/tests/testthat/test-crew_controller_local.R index 2a0a0e8c..f8ce6d58 100644 --- a/tests/testthat/test-crew_controller_local.R +++ b/tests/testthat/test-crew_controller_local.R @@ -15,6 +15,7 @@ crew_test("crew_controller_local()", { expect_false(x$started()) expect_null(x$summary()) expect_null(x$autoscaling) + expect_equal(length(x$pids()), 1L) x$start() expect_true(x$empty()) expect_false(x$saturated()) @@ -45,6 +46,7 @@ crew_test("crew_controller_local()", { expect_equal(s$tasks, 0L) expect_true(x$client$started) expect_true(x$started()) + expect_equal(length(x$pids()), 2L) # first task expect_equal(x$pushed, 0L) expect_equal(x$popped, 0L) diff --git a/vignettes/introduction.Rmd b/vignettes/introduction.Rmd index 6aad8fb6..94f03d2f 100644 --- a/vignettes/introduction.Rmd +++ b/vignettes/introduction.Rmd @@ -199,7 +199,7 @@ controller$client$summary() # Resources -`crew` utilizes background processes, and it may consume a lot of memory for big data workloads. A common cause of crashes is running out of computer memory. If you are running `crew` in a [`targets`](https://docs.ropensci.org/targets) pipeline (as explained [here in the `targets` user manual](https://books.ropensci.org/targets/crew.html)), consider setting `storage = "worker"` and `retrieval = "worker` in `tar_option_set()` to minimize memory consumption of the local processes (see also the [performance chapter](https://books.ropensci.org/targets/performance.html)). +`crew` utilizes background processes, and it may consume a lot of memory for big data workloads. A common cause of crashes is running out of computer memory. If you are running `crew` in a [`targets`](https://docs.ropensci.org/targets/) pipeline (as explained [here in the `targets` user manual](https://books.ropensci.org/targets/crew.html)), consider setting `storage = "worker"` and `retrieval = "worker` in `tar_option_set()` to minimize memory consumption of the local processes (see also the [performance chapter](https://books.ropensci.org/targets/performance.html)). As of `crew` version 0.9.5.9007, you monitor resources by supplying a log file path to the `log_resources` argument of the controller. That way, normal controller methods write to the log as a side effect every `seconds_interval` seconds. The output file is a comma-separated values (CSV) file which can be read into R with `readr::read_csv()`. diff --git a/vignettes/risks.Rmd b/vignettes/risks.Rmd index 974257a6..44730f12 100644 --- a/vignettes/risks.Rmd +++ b/vignettes/risks.Rmd @@ -25,7 +25,7 @@ In the event of a poorly-timed crash or network error, these processes may not t ## Crashes -The local process or [`mirai`](https://github.com/shikokuchuo/mirai) dispatcher process could crash. A common cause of crashes is running out of computer memory. The "Resources" section of the [introduction](https://wlandau.github.io/crew/articles/introduction.html) explains how to monitor memory usage. If you are running `crew` in a [`targets`](https://docs.ropensci.org/targets) pipeline (as explained [here in the `targets` user manual](https://books.ropensci.org/targets/crew.html)), consider setting `storage = "worker"` and `retrieval = "worker` in `tar_option_set()` to minimize memory consumption of the local processes (see also the [performance chapter](https://books.ropensci.org/targets/performance.html)). +The local process or [`mirai`](https://github.com/shikokuchuo/mirai) dispatcher process could crash. A common cause of crashes is running out of computer memory. The "Resources" section of the [introduction](https://wlandau.github.io/crew/articles/introduction.html) explains how to monitor memory usage. If you are running `crew` in a [`targets`](https://docs.ropensci.org/targets/) pipeline (as explained [here in the `targets` user manual](https://books.ropensci.org/targets/crew.html)), consider setting `storage = "worker"` and `retrieval = "worker` in `tar_option_set()` to minimize memory consumption of the local processes (see also the [performance chapter](https://books.ropensci.org/targets/performance.html)). In addition, `crew` worker processes may crash silently at runtime, or they may fail to launch or connect at all. The reasons may be platform-specific, but here are some common possibilities: