-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
91619f5
commit 62c2b9b
Showing
24 changed files
with
850 additions
and
2,218 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,166 +1,76 @@ | ||
################################################################################ | ||
############################## BEETHOVEN ############################# | ||
##### Main file controlling the settings, options, and sourcing of targets | ||
##### for the beethoven analysis pipeline. | ||
|
||
################################################################################ | ||
############################## LIBRARIES ############################# | ||
library(targets) | ||
library(tarchetypes) | ||
library(dplyr) | ||
library(crew) | ||
library(future) | ||
library(crew.cluster) | ||
library(beethoven) | ||
library(amadeus) | ||
|
||
|
||
# targets store location corresponds to _targets/ in the root of the project | ||
tar_config_set( | ||
store = "/opt/_targets" | ||
library(dplyr) | ||
# library(tidymodels) | ||
# library(bonsai) | ||
|
||
################################################################################ | ||
############################# CONTROLLER ############################# | ||
default_controller <- crew::crew_controller_local( | ||
name = "default_controller", | ||
workers = 4, | ||
seconds_idle = 30 | ||
) | ||
|
||
# crew contollers | ||
# For now, one is set, but we can explore the use of multiple controllers | ||
# Can also explore making the workers input for bash script or Rscript | ||
geo_controller <- crew_controller_local( | ||
name = "geo_controller", | ||
workers = 16L, | ||
launch_max = 8L, | ||
seconds_idle = 120 | ||
calc_controller <- crew::crew_controller_local( | ||
name = "calc_controller", | ||
workers = 200, | ||
seconds_idle = 30 | ||
) | ||
|
||
################################################################################ | ||
############################## STORE ############################## | ||
targets::tar_config_set(store = "_targets") | ||
|
||
|
||
# Setting up the NASA Earthdata token inside the container | ||
# This needs to be tested | ||
if (!nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN"))){ | ||
tar_source("/mnt/NASA_token_setup.R") | ||
file.exists(".netrc") | ||
file.exists(".urs_cookies") | ||
file.exists(".dodsrc") | ||
} | ||
|
||
|
||
arglist_download <- | ||
set_args_download( | ||
char_period = c("2018-01-01", "2022-12-31"), | ||
char_input_dir = "/input", | ||
nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), | ||
mod06_filelist = "/pipeline/targets/mod06_links_2018_2022.csv", | ||
export = TRUE, | ||
path_export = "/pipeline/targets/download_spec.qs" | ||
) | ||
|
||
|
||
|
||
|
||
|
||
|
||
### NOTE: It is important to source the scipts after the global variables are defined from the set_args functions | ||
#tar_source("/pipeline/targets/targets_aqs.R") | ||
tar_source("/pipeline/targets/targets_download.R") | ||
|
||
# Toy test files - note we will not have functions defined like this directly in | ||
# the _targets.R file | ||
my_fun_a <- function(n) { | ||
rnorm(n) | ||
} | ||
|
||
my_fun_b <- function(x) { | ||
x^2 | ||
} | ||
|
||
|
||
|
||
|
||
tar_option_set( | ||
packages = | ||
c( "amadeus", "targets", "tarchetypes", | ||
"data.table", "sf", "terra", "exactextractr", | ||
"dplyr", "qs", "callr", "stars", "rlang"), | ||
controller = crew_controller_group(geo_controller), | ||
resources = tar_resources( | ||
crew = tar_resources_crew(controller = "geo_controller") | ||
), | ||
error = "abridge", | ||
################################################################################ | ||
############################## OPTIONS ############################## | ||
targets::tar_option_set( | ||
packages = c( | ||
"beethoven", "targets", "tarchetypes", "dplyr", | ||
"data.table", "sf", "crew", "crew.cluster" | ||
), | ||
repository = "local", | ||
error = "continue", | ||
memory = "transient", | ||
format = "qs", | ||
storage = "worker", | ||
deployment = "worker", | ||
garbage_collection = TRUE, | ||
seed = 202401L | ||
) | ||
|
||
list( | ||
tar_target(name = A, command = my_fun_a(100)), | ||
tar_target(name = B, command = my_fun_b(A), pattern = A), | ||
tar_target(name = save_input, command = saveRDS(B, "/input/input.rds")), | ||
tar_target( # Test download data with amadeus | ||
download_test, | ||
amadeus::download_narr( | ||
variables = c("weasd", "omega"), | ||
year = c(2023, 2023), | ||
directory_to_save = "/input/narr_monolevel", | ||
acknowledgement = TRUE, | ||
download = TRUE, | ||
remove_command = TRUE | ||
) | ||
), | ||
target_download | ||
seed = 202401L, | ||
controller = crew_controller_group( | ||
default_controller, | ||
calc_controller | ||
) | ||
) | ||
|
||
################################################################################ | ||
########################### SOURCE TARGETS ########################### | ||
targets::tar_source("inst/targets/targets_arglist.R") | ||
# targets::tar_source("inst/targets/targets_aqs.R") | ||
# targets::tar_source("inst/targets/targets_download.R") | ||
# targets::tar_source("inst/targets/targets_calculate_fit.R") | ||
|
||
################################################################################ | ||
############################## DOWNLOAD ############################## | ||
Sys.setenv("BTV_DOWNLOAD_PASS" = "TRUE") | ||
if (Sys.getenv("BTV_DOWNLOAD_PASS") == "TRUE") { | ||
target_download <- NULL | ||
} | ||
|
||
# Style below that uses sources scripts for targets by pipeline step | ||
# Note that variables created in _targets.R are in the same local | ||
# environment as the sourced scripts | ||
|
||
# list( | ||
# target_init, | ||
# target_download | ||
# target_calculate_fit, | ||
# target_baselearner#, | ||
# target_metalearner, | ||
# target_calculate_predict, | ||
# target_predict, | ||
# # documents and summary statistics | ||
# targets::tar_target( | ||
# summary_urban_rural, | ||
# summary_prediction( | ||
# grid_filled, | ||
# level = "point", | ||
# contrast = "urbanrural")) | ||
# , | ||
# targets::tar_target( | ||
# summary_state, | ||
# summary_prediction( | ||
# grid_filled, | ||
# level = "point", | ||
# contrast = "state" | ||
# ) | ||
# ) | ||
# ) | ||
|
||
# targets::tar_visnetwork(targets_only = TRUE) | ||
# END OF FILE | ||
|
||
# list( | ||
# target_init, | ||
# target_download, | ||
# target_calculate_fit, | ||
# target_baselearner, | ||
# target_metalearner, | ||
# target_calculate_predict#, | ||
# target_predict, | ||
# # documents and summary statistics | ||
# targets::tar_target( | ||
# summary_urban_rural, | ||
# summary_prediction( | ||
# grid_filled, | ||
# level = "point", | ||
# contrast = "urbanrural")) | ||
# , | ||
# targets::tar_target( | ||
# summary_state, | ||
# summary_prediction( | ||
# grid_filled, | ||
# level = "point", | ||
# contrast = "state" | ||
# ) | ||
# ) | ||
################################################################################ | ||
############################## PIPELINE ############################## | ||
list( | ||
target_arglist | ||
# target_aqs | ||
# target_download, | ||
# target_calculate_fit | ||
) | ||
|
||
# targets::tar_visnetwork(targets_only = TRUE) | ||
# END OF FILE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,3 @@ | ||
main: | ||
store: /ddn/gs1/home/manwareme/beethoven/beethoven_targets | ||
store: _targets | ||
inherits: _targets |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.