HPC Episode

config.yaml

@@ -69,6 +69,7 @@ episodes:
 - branch.Rmd
 - parallel.Rmd
 - quarto.Rmd
+- hpc.Rmd
 
 # Information for Learners
 learners: 

episodes/files/plans/plan_slurm.R

@@ -0,0 +1,49 @@
+library(crew.cluster)
+library(targets)
+library(tarchetypes)
+library(palmerpenguins)
+library(broom)
+suppressPackageStartupMessages(library(tidyverse))
+
+source("R/packages.R")
+source("R/functions.R")
+
+tar_option_set(
+  controller = crew_controller_slurm(
+    workers = 3,
+    script_lines = "module load R",
+    slurm_memory_gigabytes_per_cpu = 1
+  )
+)
+
+tar_plan(
+  # Load raw data
+  tar_file_read(
+    penguins_data_raw,
+    path_to_file("penguins_raw.csv"),
+    read_csv(!!.x, show_col_types = FALSE)
+  ),
+  # Clean data
+  penguins_data = clean_penguin_data(penguins_data_raw),
+  # Build models
+  models = list(
+    combined_model = lm(
+      bill_depth_mm ~ bill_length_mm, data = penguins_data),
+    species_model = lm(
+      bill_depth_mm ~ bill_length_mm + species, data = penguins_data),
+    interaction_model = lm(
+      bill_depth_mm ~ bill_length_mm * species, data = penguins_data)
+  ),
+  # Get model summaries
+  tar_target(
+    model_summaries,
+    glance_with_mod_name_slow(models),
+    pattern = map(models)
+  ),
+  # Get model predictions
+  tar_target(
+    model_predictions,
+    augment_with_mod_name_slow(models),
+    pattern = map(models)
+  )
+)

episodes/files/plans/plan_slurm_gpu.R

@@ -0,0 +1,52 @@
+graphics_devices <- function(){
+  system2("lshw", c("-class", "display"), stdout=TRUE, stderr=FALSE)
+}
+
+library(crew)
+library(targets)
+library(tarchetypes)
+library(crew.cluster)
+
+source("R/packages.R")
+source("R/functions.R")
+
+tar_option_set(
+  controller = crew_controller_group(
+    crew_controller_slurm(
+      name = "cpu_worker",
+      workers = 1,
+      script_lines = "module load R",
+      slurm_memory_gigabytes_per_cpu = 1,
+      slurm_cpus_per_task = 1
+    ),
+
+    crew_controller_slurm(
+      name = "gpu_worker",
+      workers = 1,
+      script_lines = c(
+        "#SBATCH --partition=gpuq",
+        "#SBATCH --gres=gpu:1",
+        "module load R"
+      ),
+      slurm_memory_gigabytes_per_cpu = 1,
+      slurm_cpus_per_task = 1
+    )
+  )
+)
+
+tar_plan(
+  tar_target(
+    cpu_hardware,
+    graphics_devices(),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "cpu_worker")
+    )
+  ),
+  tar_target(
+    gpu_hardware,
+    graphics_devices(),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "gpu_worker")
+    )
+  )
+)

episodes/files/plans/plan_slurm_memory.R

@@ -0,0 +1,39 @@
+library(crew)
+library(targets)
+library(tarchetypes)
+library(crew.cluster)
+
+source("R/packages.R")
+source("R/functions.R")
+
+small_memory <- crew_controller_slurm(
+  name = "small_memory",
+  script_lines = "module load R",
+  slurm_memory_gigabytes_per_cpu = 10
+)
+big_memory <- crew_controller_slurm(
+  name = "big_memory",
+  script_lines = "module load R",
+  slurm_memory_gigabytes_per_cpu = 20
+)
+
+tar_option_set(
+  controller = crew_controller_group(small_memory, big_memory)
+)
+
+list(
+  tar_target(
+    name = big_memory_task,
+    command = Sys.getenv("SLURM_MEM_PER_CPU"),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "big_memory")
+    )
+  ),
+  tar_target(
+    name = small_memory_task,
+    command = Sys.getenv("SLURM_MEM_PER_CPU"),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "small_memory")
+    )
+  )
+)