From de46626d9124189ef4867fc69485b71658ce2c9a Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 3 Apr 2024 20:07:38 +0800 Subject: [PATCH 1/8] Increment version number to 3.11.0.9000 --- DESCRIPTION | 2 +- NEWS.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7e331c8..91e6427 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: tarflow.iquizoo Title: Setup "targets" Workflows for "iquizoo" Data Processing -Version: 3.11.0 +Version: 3.11.0.9000 Authors@R: c( person("Liang", "Zhang", , "psychelzh@outlook.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9041-1150")), diff --git a/NEWS.md b/NEWS.md index 3d61900..cdc8e17 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +# tarflow.iquizoo (development version) + # tarflow.iquizoo 3.11.0 ## Breaking Changes From aef9ceb1206bd867ce2d4f1d53f744338e418377 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 16:19:22 +0800 Subject: [PATCH 2/8] Modify one test case by running pipeline with `callr:r()` Signed-off-by: Liang Zhang --- tests/testthat/test-targets.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-targets.R b/tests/testthat/test-targets.R index d04473c..51bb094 100644 --- a/tests/testthat/test-targets.R +++ b/tests/testthat/test-targets.R @@ -52,9 +52,9 @@ test_that("Workflow works", { ~organization_name, ~project_name, "北京师范大学", "4.19-4.20夜晚睡眠test" ) - tar_prep_iquizoo(params, combine = "scores") + tarflow.iquizoo::tar_prep_iquizoo(params, combine = "scores") }) - targets::tar_make(reporter = "silent", callr_function = NULL) + expect_silent(targets::tar_make(reporter = "silent")) expect_snapshot_value(targets::tar_objects(), style = "json2") expect_snapshot_value(targets::tar_read(users), style = "json2") expect_snapshot_value(targets::tar_read(scores), style = "json2") From 8373ef01299ad420903211722705a0ae7da94d6c Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 16:21:50 +0800 Subject: [PATCH 3/8] Separate `tar_prep_proj()` into `tar_prep_hash()` and `tar_fetch_users()` Signed-off-by: Liang Zhang --- NAMESPACE | 3 +- R/targets.R | 81 +++++++++++++++++++++++------------------- _pkgdown.yml | 3 +- man/tar_fetch_users.Rd | 21 +++++++++++ man/tar_prep_hash.Rd | 25 +++++++++++++ man/tar_prep_proj.Rd | 29 --------------- 6 files changed, 94 insertions(+), 68 deletions(-) create mode 100644 man/tar_fetch_users.Rd create mode 100644 man/tar_prep_hash.Rd delete mode 100644 man/tar_prep_proj.Rd diff --git a/NAMESPACE b/NAMESPACE index 602bfd6..3423cb1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,8 +9,9 @@ export(setup_option_file) export(setup_source) export(setup_templates) export(tar_fetch_data) +export(tar_fetch_users) +export(tar_prep_hash) export(tar_prep_iquizoo) -export(tar_prep_proj) export(tar_prep_raw) export(use_targets_pipeline) import(rlang) diff --git a/R/targets.R b/R/targets.R index 9f204d9..991dfc3 100644 --- a/R/targets.R +++ b/R/targets.R @@ -78,10 +78,14 @@ tar_prep_iquizoo <- function(params, ..., "contents_origin", expr(unserialize(!!serialize(contents, NULL))) ), - tar_prep_proj(contents, templates, check_progress), + if (check_progress) tar_prep_hash(contents, templates), + tar_fetch_users(contents, templates), sapply( what, - \(what) tar_fetch_data(contents, templates, what, check_progress), + tar_fetch_data, + contents = contents, + templates = templates, + check_progress = check_progress, simplify = FALSE ), if ("raw_data" %in% what && action_raw_data != "none") { @@ -105,57 +109,60 @@ tar_prep_iquizoo <- function(params, ..., ) } -#' Generate a set of targets for preparing project-level data +#' Generate a set of targets for fetching progress hash #' -#' There are mainly two types of data to be fetched, i.e., the progress hash and -#' the user information. The former is used to check the progress of the -#' project, while the latter is used to identify the users involved in the -#' project. +#' The progress hash stores the progress of the project, which is used to check +#' whether the project is updated. +#' +#' These objects are named as `progress_hash_{project_id}` for each project. #' #' @param contents The contents structure used as the configuration of data #' fetching. #' @param templates The SQL template files used to fetch data. See #' [setup_templates()] for details. -#' @param check_progress Whether to check the progress hash. When set as `TRUE`, -#' a progress hash objects named as `progress_hash_{project_id}` for each -#' project will be added into the target list. Set it as `FALSE` if the -#' projects are finalized. #' @return A list of target objects. #' @export -tar_prep_proj <- function(contents, - templates = setup_templates(), - check_progress = TRUE) { - c( - if (check_progress) { - tarchetypes::tar_map( - data.frame(project_id = as.character(unique(contents$project_id))), - targets::tar_target_raw( - "progress_hash", - bquote( - fetch_iquizoo( - .(read_file(templates[["progress_hash"]])), - params = list(project_id) - ) - ), - packages = "tarflow.iquizoo", - cue = targets::tar_cue("always") - ) - ) - }, +tar_prep_hash <- function(contents, templates = setup_templates()) { + tarchetypes::tar_map( + data.frame(project_id = as.character(unique(contents$project_id))), targets::tar_target_raw( - "users", + "progress_hash", bquote( fetch_iquizoo( - .(read_file(templates[["users"]])), - params = list(.(unique(contents$project_id))) - ) |> - unique() + .(read_file(templates[["progress_hash"]])), + params = list(project_id) + ) ), - packages = "tarflow.iquizoo" + packages = "tarflow.iquizoo", + cue = targets::tar_cue("always") ) ) } +#' Generate a set of targets for fetching user information +#' +#' The user information is used to identify the users involved in the project. +#' +#' @param contents The contents structure used as the configuration of data +#' fetching. +#' @param templates The SQL template files used to fetch data. See +#' [setup_templates()] for details. +#' @return A list of target objects. +#' @export +tar_fetch_users <- function(contents, templates = setup_templates()) { + targets::tar_target_raw( + "users", + bquote( + fetch_iquizoo( + .(read_file(templates[["users"]])), + params = list(.(unique(contents$project_id))) + ) |> + unique() + ), + packages = "tarflow.iquizoo" + ) +} + #' Generate a set of targets for fetching data #' #' This target factory is the main part of the `tar_prep_iquizoo` function. It diff --git a/_pkgdown.yml b/_pkgdown.yml index bab4757..f3e8a52 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -7,9 +7,10 @@ reference: contents: - use_targets_pipeline - tar_prep_iquizoo - - tar_prep_proj + - tar_fetch_users - tar_fetch_data - tar_prep_raw + - tar_prep_hash - title: "Low-level Database operations" desc: Functions to help you interact with database. contents: diff --git a/man/tar_fetch_users.Rd b/man/tar_fetch_users.Rd new file mode 100644 index 0000000..ada5c01 --- /dev/null +++ b/man/tar_fetch_users.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/targets.R +\name{tar_fetch_users} +\alias{tar_fetch_users} +\title{Generate a set of targets for fetching user information} +\usage{ +tar_fetch_users(contents, templates = setup_templates()) +} +\arguments{ +\item{contents}{The contents structure used as the configuration of data +fetching.} + +\item{templates}{The SQL template files used to fetch data. See +\code{\link[=setup_templates]{setup_templates()}} for details.} +} +\value{ +A list of target objects. +} +\description{ +The user information is used to identify the users involved in the project. +} diff --git a/man/tar_prep_hash.Rd b/man/tar_prep_hash.Rd new file mode 100644 index 0000000..54cf12e --- /dev/null +++ b/man/tar_prep_hash.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/targets.R +\name{tar_prep_hash} +\alias{tar_prep_hash} +\title{Generate a set of targets for fetching progress hash} +\usage{ +tar_prep_hash(contents, templates = setup_templates()) +} +\arguments{ +\item{contents}{The contents structure used as the configuration of data +fetching.} + +\item{templates}{The SQL template files used to fetch data. See +\code{\link[=setup_templates]{setup_templates()}} for details.} +} +\value{ +A list of target objects. +} +\description{ +The progress hash stores the progress of the project, which is used to check +whether the project is updated. +} +\details{ +These objects are named as \verb{progress_hash_\{project_id\}} for each project. +} diff --git a/man/tar_prep_proj.Rd b/man/tar_prep_proj.Rd deleted file mode 100644 index 17b2101..0000000 --- a/man/tar_prep_proj.Rd +++ /dev/null @@ -1,29 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/targets.R -\name{tar_prep_proj} -\alias{tar_prep_proj} -\title{Generate a set of targets for preparing project-level data} -\usage{ -tar_prep_proj(contents, templates = setup_templates(), check_progress = TRUE) -} -\arguments{ -\item{contents}{The contents structure used as the configuration of data -fetching.} - -\item{templates}{The SQL template files used to fetch data. See -\code{\link[=setup_templates]{setup_templates()}} for details.} - -\item{check_progress}{Whether to check the progress hash. When set as \code{TRUE}, -a progress hash objects named as \verb{progress_hash_\{project_id\}} for each -project will be added into the target list. Set it as \code{FALSE} if the -projects are finalized.} -} -\value{ -A list of target objects. -} -\description{ -There are mainly two types of data to be fetched, i.e., the progress hash and -the user information. The former is used to check the progress of the -project, while the latter is used to identify the users involved in the -project. -} From da1de1ec81b334a677c176f720e06fe8fb25a454 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 16:41:04 +0800 Subject: [PATCH 4/8] NEWS bulletin Signed-off-by: Liang Zhang --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index cdc8e17..25355ab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # tarflow.iquizoo (development version) +* Separate `tar_prep_proj()` into `tar_prep_hash()` and `tar_fetch_users()`. + # tarflow.iquizoo 3.11.0 ## Breaking Changes From 510f4b924c13a5be4fc9ba9bc7d13fd135747080 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 16:55:15 +0800 Subject: [PATCH 5/8] Use `lapply()` Signed-off-by: Liang Zhang --- R/targets.R | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/R/targets.R b/R/targets.R index 991dfc3..c666174 100644 --- a/R/targets.R +++ b/R/targets.R @@ -123,19 +123,21 @@ tar_prep_iquizoo <- function(params, ..., #' @return A list of target objects. #' @export tar_prep_hash <- function(contents, templates = setup_templates()) { - tarchetypes::tar_map( - data.frame(project_id = as.character(unique(contents$project_id))), - targets::tar_target_raw( - "progress_hash", - bquote( - fetch_iquizoo( - .(read_file(templates[["progress_hash"]])), - params = list(project_id) - ) - ), - packages = "tarflow.iquizoo", - cue = targets::tar_cue("always") - ) + lapply( + as.character(unique(contents$project_id)), + \(project_id) { + targets::tar_target_raw( + paste0("progress_hash_", project_id), + bquote( + fetch_iquizoo( + .(read_file(templates[["progress_hash"]])), + params = list(.(project_id)) + ) + ), + packages = "tarflow.iquizoo", + cue = targets::tar_cue("always") + ) + } ) } From d37d406b3162fb4b43747d23a2762e492aab6f93 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 16:59:11 +0800 Subject: [PATCH 6/8] Ensure templates are created by `setup_templates()` Signed-off-by: Liang Zhang --- R/setup.R | 9 +++++++++ R/targets.R | 14 ++++++-------- man/tar_fetch_data.Rd | 6 +++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/R/setup.R b/R/setup.R index 0c184a6..fd52d05 100644 --- a/R/setup.R +++ b/R/setup.R @@ -129,3 +129,12 @@ default_file <- function() { } # nocov end + +check_templates <- function(templates) { + if (!inherits(templates, "tarflow.template")) { + cli::cli_abort( + "{.arg templates} must be created by {.fun setup_templates}.", + class = "tarflow_bad_templates" + ) + } +} diff --git a/R/targets.R b/R/targets.R index c666174..bd68c47 100644 --- a/R/targets.R +++ b/R/targets.R @@ -45,12 +45,7 @@ tar_prep_iquizoo <- function(params, ..., templates = setup_templates(), check_progress = TRUE) { check_dots_empty() - if (!inherits(templates, "tarflow.template")) { - cli::cli_abort( - "{.arg templates} must be created by {.fun setup_templates}.", - class = "tarflow_bad_templates" - ) - } + check_templates(templates) what <- match.arg(what, several.ok = TRUE) action_raw_data <- match.arg(action_raw_data) if (!is.null(combine) && !all(combine %in% objects())) { @@ -123,6 +118,7 @@ tar_prep_iquizoo <- function(params, ..., #' @return A list of target objects. #' @export tar_prep_hash <- function(contents, templates = setup_templates()) { + check_templates(templates) lapply( as.character(unique(contents$project_id)), \(project_id) { @@ -152,6 +148,7 @@ tar_prep_hash <- function(contents, templates = setup_templates()) { #' @return A list of target objects. #' @export tar_fetch_users <- function(contents, templates = setup_templates()) { + check_templates(templates) targets::tar_target_raw( "users", bquote( @@ -172,9 +169,9 @@ tar_fetch_users <- function(contents, templates = setup_templates()) { #' #' @param contents The contents structure used as the configuration of data #' fetching. +#' @param what What to fetch. #' @param templates The SQL template files used to fetch data. See #' [setup_templates()] for details. -#' @param what What to fetch. #' @param check_progress Whether to check the progress hash. If set as `TRUE`, #' Before fetching the data, the progress hash objects named as #' `progress_hash_{project_id}` will be depended on, which are typically @@ -183,10 +180,11 @@ tar_fetch_users <- function(contents, templates = setup_templates()) { #' @return A list of target objects. #' @export tar_fetch_data <- function(contents, - templates = setup_templates(), what = c("raw_data", "scores"), + templates = setup_templates(), check_progress = TRUE) { what <- match.arg(what) + check_templates(templates) by( contents, contents$game_id, diff --git a/man/tar_fetch_data.Rd b/man/tar_fetch_data.Rd index 8a52434..aedc4af 100644 --- a/man/tar_fetch_data.Rd +++ b/man/tar_fetch_data.Rd @@ -6,8 +6,8 @@ \usage{ tar_fetch_data( contents, - templates = setup_templates(), what = c("raw_data", "scores"), + templates = setup_templates(), check_progress = TRUE ) } @@ -15,11 +15,11 @@ tar_fetch_data( \item{contents}{The contents structure used as the configuration of data fetching.} +\item{what}{What to fetch.} + \item{templates}{The SQL template files used to fetch data. See \code{\link[=setup_templates]{setup_templates()}} for details.} -\item{what}{What to fetch.} - \item{check_progress}{Whether to check the progress hash. If set as \code{TRUE}, Before fetching the data, the progress hash objects named as \verb{progress_hash_\{project_id\}} will be depended on, which are typically From ca093412c953473b9d5d1871944336d3d9ffdc8a Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 4 Apr 2024 17:23:05 +0800 Subject: [PATCH 7/8] Fix one link Signed-off-by: Liang Zhang --- R/targets.R | 2 +- man/tar_fetch_data.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/targets.R b/R/targets.R index bd68c47..63a713a 100644 --- a/R/targets.R +++ b/R/targets.R @@ -175,7 +175,7 @@ tar_fetch_users <- function(contents, templates = setup_templates()) { #' @param check_progress Whether to check the progress hash. If set as `TRUE`, #' Before fetching the data, the progress hash objects named as #' `progress_hash_{project_id}` will be depended on, which are typically -#' generated by [tar_prep_proj()]. If the projects are finalized, set this +#' generated by [tar_prep_hash()]. If the projects are finalized, set this #' argument as `FALSE`. #' @return A list of target objects. #' @export diff --git a/man/tar_fetch_data.Rd b/man/tar_fetch_data.Rd index aedc4af..50b07b4 100644 --- a/man/tar_fetch_data.Rd +++ b/man/tar_fetch_data.Rd @@ -23,7 +23,7 @@ fetching.} \item{check_progress}{Whether to check the progress hash. If set as \code{TRUE}, Before fetching the data, the progress hash objects named as \verb{progress_hash_\{project_id\}} will be depended on, which are typically -generated by \code{\link[=tar_prep_proj]{tar_prep_proj()}}. If the projects are finalized, set this +generated by \code{\link[=tar_prep_hash]{tar_prep_hash()}}. If the projects are finalized, set this argument as \code{FALSE}.} } \value{ From fd41e334faea954809cdc6072c99fb9068241140 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sat, 6 Apr 2024 10:43:51 +0800 Subject: [PATCH 8/8] Increment version number to 3.11.1 --- DESCRIPTION | 2 +- NEWS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 91e6427..223f94e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: tarflow.iquizoo Title: Setup "targets" Workflows for "iquizoo" Data Processing -Version: 3.11.0.9000 +Version: 3.11.1 Authors@R: c( person("Liang", "Zhang", , "psychelzh@outlook.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9041-1150")), diff --git a/NEWS.md b/NEWS.md index 25355ab..1190934 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -# tarflow.iquizoo (development version) +# tarflow.iquizoo 3.11.1 * Separate `tar_prep_proj()` into `tar_prep_hash()` and `tar_fetch_users()`.