Skip to content

Commit

Permalink
migrate dev/refactor/ -> root/
Browse files Browse the repository at this point in the history
  • Loading branch information
mitchellmanware committed Oct 3, 2024
1 parent 91619f5 commit 62c2b9b
Show file tree
Hide file tree
Showing 24 changed files with 850 additions and 2,218 deletions.
10 changes: 5 additions & 5 deletions dev/helpers.R → R/helpers.R
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# srun --partition=geo --cpus-per-task=1 --pty top
# scontrol show node gn040815 # geo cluster
# scontrol show node cn040827 # hihgmem cluster
# sacct -j 303493 --format=JobID,Elapsed,TotalCPU,MaxRSS
# Helper functions for checking SLURM jobs and nodes

# nocov start
job <- function(job_id) {
system(
paste0("sacct -j ", job_id, " --format=JobID,Elapsed,TotalCPU,MaxRSS")
Expand Down Expand Up @@ -31,4 +30,5 @@ cancel <- function() {

batch <- function(file = "run.sh") {
system(paste0("sbatch ", file))
}
}
# nocov end
208 changes: 59 additions & 149 deletions _targets.R
100755 → 100644
Original file line number Diff line number Diff line change
@@ -1,166 +1,76 @@
################################################################################
############################## BEETHOVEN #############################
##### Main file controlling the settings, options, and sourcing of targets
##### for the beethoven analysis pipeline.

################################################################################
############################## LIBRARIES #############################
library(targets)
library(tarchetypes)
library(dplyr)
library(crew)
library(future)
library(crew.cluster)
library(beethoven)
library(amadeus)


# targets store location corresponds to _targets/ in the root of the project
tar_config_set(
store = "/opt/_targets"
library(dplyr)
# library(tidymodels)
# library(bonsai)

################################################################################
############################# CONTROLLER #############################
default_controller <- crew::crew_controller_local(
name = "default_controller",
workers = 4,
seconds_idle = 30
)

# crew contollers
# For now, one is set, but we can explore the use of multiple controllers
# Can also explore making the workers input for bash script or Rscript
geo_controller <- crew_controller_local(
name = "geo_controller",
workers = 16L,
launch_max = 8L,
seconds_idle = 120
calc_controller <- crew::crew_controller_local(
name = "calc_controller",
workers = 200,
seconds_idle = 30
)

################################################################################
############################## STORE ##############################
targets::tar_config_set(store = "_targets")


# Setting up the NASA Earthdata token inside the container
# This needs to be tested
if (!nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN"))){
tar_source("/mnt/NASA_token_setup.R")
file.exists(".netrc")
file.exists(".urs_cookies")
file.exists(".dodsrc")
}


arglist_download <-
set_args_download(
char_period = c("2018-01-01", "2022-12-31"),
char_input_dir = "/input",
nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"),
mod06_filelist = "/pipeline/targets/mod06_links_2018_2022.csv",
export = TRUE,
path_export = "/pipeline/targets/download_spec.qs"
)






### NOTE: It is important to source the scipts after the global variables are defined from the set_args functions
#tar_source("/pipeline/targets/targets_aqs.R")
tar_source("/pipeline/targets/targets_download.R")

# Toy test files - note we will not have functions defined like this directly in
# the _targets.R file
my_fun_a <- function(n) {
rnorm(n)
}

my_fun_b <- function(x) {
x^2
}




tar_option_set(
packages =
c( "amadeus", "targets", "tarchetypes",
"data.table", "sf", "terra", "exactextractr",
"dplyr", "qs", "callr", "stars", "rlang"),
controller = crew_controller_group(geo_controller),
resources = tar_resources(
crew = tar_resources_crew(controller = "geo_controller")
),
error = "abridge",
################################################################################
############################## OPTIONS ##############################
targets::tar_option_set(
packages = c(
"beethoven", "targets", "tarchetypes", "dplyr",
"data.table", "sf", "crew", "crew.cluster"
),
repository = "local",
error = "continue",
memory = "transient",
format = "qs",
storage = "worker",
deployment = "worker",
garbage_collection = TRUE,
seed = 202401L
)

list(
tar_target(name = A, command = my_fun_a(100)),
tar_target(name = B, command = my_fun_b(A), pattern = A),
tar_target(name = save_input, command = saveRDS(B, "/input/input.rds")),
tar_target( # Test download data with amadeus
download_test,
amadeus::download_narr(
variables = c("weasd", "omega"),
year = c(2023, 2023),
directory_to_save = "/input/narr_monolevel",
acknowledgement = TRUE,
download = TRUE,
remove_command = TRUE
)
),
target_download
seed = 202401L,
controller = crew_controller_group(
default_controller,
calc_controller
)
)

################################################################################
########################### SOURCE TARGETS ###########################
targets::tar_source("inst/targets/targets_arglist.R")
# targets::tar_source("inst/targets/targets_aqs.R")
# targets::tar_source("inst/targets/targets_download.R")
# targets::tar_source("inst/targets/targets_calculate_fit.R")

################################################################################
############################## DOWNLOAD ##############################
Sys.setenv("BTV_DOWNLOAD_PASS" = "TRUE")
if (Sys.getenv("BTV_DOWNLOAD_PASS") == "TRUE") {
target_download <- NULL
}

# Style below that uses sources scripts for targets by pipeline step
# Note that variables created in _targets.R are in the same local
# environment as the sourced scripts

# list(
# target_init,
# target_download
# target_calculate_fit,
# target_baselearner#,
# target_metalearner,
# target_calculate_predict,
# target_predict,
# # documents and summary statistics
# targets::tar_target(
# summary_urban_rural,
# summary_prediction(
# grid_filled,
# level = "point",
# contrast = "urbanrural"))
# ,
# targets::tar_target(
# summary_state,
# summary_prediction(
# grid_filled,
# level = "point",
# contrast = "state"
# )
# )
# )

# targets::tar_visnetwork(targets_only = TRUE)
# END OF FILE

# list(
# target_init,
# target_download,
# target_calculate_fit,
# target_baselearner,
# target_metalearner,
# target_calculate_predict#,
# target_predict,
# # documents and summary statistics
# targets::tar_target(
# summary_urban_rural,
# summary_prediction(
# grid_filled,
# level = "point",
# contrast = "urbanrural"))
# ,
# targets::tar_target(
# summary_state,
# summary_prediction(
# grid_filled,
# level = "point",
# contrast = "state"
# )
# )
################################################################################
############################## PIPELINE ##############################
list(
target_arglist
# target_aqs
# target_download,
# target_calculate_fit
)

# targets::tar_visnetwork(targets_only = TRUE)
# END OF FILE
3 changes: 2 additions & 1 deletion _targets.yaml
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
main:
store: /ddn/gs1/home/manwareme/beethoven/beethoven_targets
store: _targets
inherits: _targets
14 changes: 10 additions & 4 deletions container/beethoven_dl_calc.def
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ From: rocker/geospatial:latest
locale-gen en_US.UTF-8

# Install fonts for Unicode support
apt-get install -y fonts-dejavu fonts-liberation fonts-noto fonts-unifont
apt-get install -y fonts-dejavu fonts-liberation fonts-noto \
fonts-unifont

# Set locale for the environment
echo "LANG=en_US.UTF-8" >> /etc/default/locale
Expand All @@ -21,13 +22,17 @@ From: rocker/geospatial:latest
# Install SLURM client for job submission
apt-get install -y slurm-client

# Create directories
mkdir /pipeline
mkdir /input
mkdir /opt/_targets

Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster','testthat','amadeus'))"
# Install R packages
Rscript -e "install.packages('ps', version = '1.8.0')"
Rscript -e "install.packages(c('pak', 'targets', 'tarchetypes', 'crew', \
'crew.cluster', 'testthat', 'amadeus', 'tidymodels', 'bonsai'))"
Rscript -e "pak::pak('NIEHS/beethoven@dev-0930')"

%environment
# Set locale for the container environment
export LANG=en_US.UTF-8
Expand All @@ -37,4 +42,5 @@ From: rocker/geospatial:latest
%runscript

%labels
basic geospatial with targets and crew plus unicode text so the target progress prints nicely
basic geospatial with targets and crew plus unicode text so the target \
progress prints nicely
Loading

0 comments on commit 62c2b9b

Please sign in to comment.