Pipeline-compact

.Rbuildignore

@@ -1,4 +1,5 @@
 ^vignettes/
+^tests/
 ^input/
 ^tools/
 ^manuscript/
@@ -12,4 +13,15 @@
 ^_pkgdown\.yml$
 ^docs$
 ^pkgdown$
-^.lintr
+^.lintr
+input/*
+inst/targets-wide
+^.future/
+^_targets/
+\.out$
+\.err$
+\.sif$
+\.log$
+\.vscode
+\.qmd$
+\.sh$

.github/workflows/check-standard.yaml

@@ -23,7 +23,14 @@ jobs:
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          extra-packages: any::rcmdcheck
+          extra-packages: any::rcmdcheck, any::remotes, any::units, any::rmarkdown
           needs: check
 
+      - name: install remote packages
+        run: |
+          Rscript -e 'remotes::install_github(sprintf("NIEHS/%s", c("amadeus", "chopin")), upgrade = FALSE)'
+        shell: bash
+
       - uses: r-lib/actions/check-r-package@v2
+        with:
+            error-on: '"error"'

.github/workflows/test-coverage.yaml

@@ -24,11 +24,16 @@ jobs:
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          extra-packages: any::covr
+          extra-packages: any::covr, any::remotes
           needs: coverage
 
+      - name: install remote packages
+        run: |
+          Rscript -e 'remotes::install_github(sprintf("NIEHS/%s", c("amadeus", "chopin")), upgrade = FALSE)'
+        shell: bash
+
       - name: Cache C++ and R dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/R
@@ -90,7 +95,7 @@ jobs:
         run: |
           npm i -g badgen-cli
           export COV=${{ steps.get-values.outputs.coverage }}
-          COLOR=$(node -p '+process.env.COV >= 95 ? `green` : `orange`')
+          COLOR=$(node -p '+process.env.COV >= 95 ? `green` : `yellow`')
           mkdir -p badges
           badgen -j coverage -s $COV% -c $COLOR > badges/coverage.svg
 

.gitignore

@@ -88,5 +88,17 @@ tools/shiny_explore_pm/missingness_exploration_pm_quarto_shiny_data/
 
 # raw data
 input
+
+# pipeline
+_targets
+
+# future batchtools outputs
 .future
-_targets
+slurm_error.log
+
+# SLURM messages and logs
+**/*.err
+**/*.out
+
+# Apptainer images
+**/*.sif

.lintr

@@ -2,5 +2,7 @@ linters: linters_with_defaults(
     commented_code_linter = NULL
   )
 exclusions: list(
-    "tests/testthat/test-download_functions.R"
+    "tests/testthat/test-download_functions.R",
+    "inst",
+    "_targets.R"
   )

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: beethoven
 Title: Building an Extensible, rEproducible, Test-driven, Harmonized, Open-source, Versioned, ENsemble model for air quality
-Version: 0.1.1
+Version: 0.3.0
 Authors@R: c(
     person("Kyle", "Messier", , "[email protected]", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9508-9623")),
     person("Insang", "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")),
@@ -15,14 +15,63 @@ Authors@R: c(
   )
 Description: Near Real Time air pollution model results and code produced by the SET group. It is fully tested, versioned, and open source and open access.
 Depends: R (>= 4.1.0)
-Imports: dplyr, sf, stats, terra, methods, BART, data.table, httr, rvest, spData, exactextractr, utils, stringr, testthat (>= 3.0.0), doParallel, parallelly, stars, foreach, future, ranger, xgboost, torch
-Suggests: covr, withr, knitr, rmarkdown, sftime, lwgeom, FNN, ggplot2, scatterpie, doRNG
+Imports:
+  dplyr,
+  sf,
+  stats,
+  terra,
+  methods,
+  BART,
+  data.table,
+  future,
+  targets,
+  tune (>= 1.2.1),
+  rlang,
+  rsample,
+  tidyr,
+  tidyselect,
+  yardstick,
+  workflows,
+  recipes,
+  missRanger,
+  parsnip,
+  future.apply,
+  future.batchtools,
+  anticlust,
+  qs,
+  stringi,
+  collapse,
+  tigris,
+  graphics
+Suggests:
+  testthat (>= 3.0.0),
+  covr,
+  withr,
+  tarchetypes,
+  tidymodels,
+  xgboost,
+  torch,
+  brulee,
+  glmnet,
+  knitr,
+  rmarkdown,
+  stars,
+  sftime,
+  lwgeom,
+  FNN,
+  ggplot2,
+  doRNG,
+  DT,
+  scatterplot3d,
+  furrr,
+  quarto
+Remotes: NIEHS/amadeus, NIEHS/chopin
 Encoding: UTF-8
 VignetteBuilder: knitr, rmarkdown
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.3.1
 Config/Needs/website: tidyverse/tidytemplate
 Config/testhat/edition: 3
 License: MIT + file LICENSE
-URL: https://github.com/Spatiotemporal-Exposures-and-Toxicology/beethoven, 
-https: //spatiotemporal-exposures-and-toxicology.github.io/beethoven/
+URL: https://github.com/NIEHS/beethoven, 
+https: //NIEHS.github.io/beethoven/

NAMESPACE

@@ -1,94 +1,114 @@
 # Generated by roxygen2: do not edit by hand
 
-export(build_pipeline)
-export(calc_covariates)
-export(calc_ecoregion)
-export(calc_koppen_geiger)
-export(calc_modis)
-export(calc_nei)
-export(calc_nlcd_ratio)
-export(calc_temporal_dummies)
-export(calc_tri)
-export(check_crs_is_valid)
-export(check_data_completeness)
-export(check_input_raster_in_extent)
-export(check_means_are_valid)
-export(check_output_locs_are_valid)
-export(check_variances_are_valid)
-export(convert_stdt)
-export(convert_stdt_sftime)
-export(convert_stdt_spatrastdataset)
-export(convert_stdt_spatvect)
-export(convert_stobj_to_stdt)
-export(generic_base_learner)
-export(generic_meta_learner)
-export(is_stdt)
+export("%tin%")
+export(add_time_col)
+export(append_predecessors)
+export(attach_xy)
+export(calc_geos_strict)
+export(calc_gmted_direct)
+export(calc_narr2)
+export(calculate)
+export(convert_cv_index_rset)
+export(df_params)
+export(divisor)
+export(fit_base_brulee)
+export(fit_base_elnet)
+export(fit_base_xgb)
+export(generate_cv_index)
+export(impute_all)
+export(inject_calculate)
+export(inject_geos)
+export(inject_gmted)
+export(inject_modis_par)
+export(load_modis_files)
+export(loadargs)
 export(meta_learner_fit)
 export(meta_learner_predict)
-export(modis_aggregate_sds)
-export(modis_get_vrt)
-export(modis_mosaic_mod06)
-export(modis_prefilter_sds)
-export(modis_preprocess_vnp46)
-export(modis_warp_stars)
-export(modis_worker)
+export(par_narr)
+export(post_calc_autojoin)
+export(post_calc_convert_time)
+export(post_calc_df_year_expand)
+export(post_calc_drop_cols)
+export(post_calc_merge_all)
+export(post_calc_merge_features)
+export(post_calc_unify_timecols)
+export(post_calc_year_expand)
+export(process_counties)
+export(process_geos_bulk)
+export(process_narr2)
+export(read_locs)
+export(read_paths)
+export(reduce_merge)
+export(search_function)
+export(set_slurm_resource)
+export(vis_rset)
 import(BART)
-import(spData)
-importClassesFrom(terra,SpatRaster)
-importClassesFrom(terra,SpatRasterDataset)
-importClassesFrom(terra,SpatVector)
+importFrom(anticlust,balanced_clustering)
+importFrom(collapse,fnth)
+importFrom(collapse,fvar)
+importFrom(collapse,join)
+importFrom(collapse,replace_inf)
+importFrom(collapse,replace_na)
+importFrom(collapse,set_collapse)
 importFrom(data.table,.SD)
+importFrom(data.table,`:=`)
 importFrom(data.table,as.data.table)
-importFrom(data.table,fread)
-importFrom(data.table,melt)
+importFrom(data.table,copy)
 importFrom(data.table,merge.data.table)
-importFrom(data.table,month)
 importFrom(data.table,rbindlist)
-importFrom(data.table,year)
-importFrom(doParallel,registerDoParallel)
+importFrom(data.table,setDTthreads)
+importFrom(data.table,setnafill)
+importFrom(dplyr,`%>%`)
+importFrom(dplyr,across)
+importFrom(dplyr,all_of)
+importFrom(dplyr,as_tibble)
 importFrom(dplyr,bind_rows)
-importFrom(dplyr,left_join)
-importFrom(exactextractr,exact_extract)
-importFrom(foreach,"%dopar%")
-importFrom(foreach,foreach)
-importFrom(future,cluster)
+importFrom(dplyr,group_by)
+importFrom(dplyr,summarize)
+importFrom(dplyr,ungroup)
+importFrom(future,multicore)
+importFrom(future,multisession)
 importFrom(future,plan)
+importFrom(future,sequential)
+importFrom(future,tweak)
+importFrom(future.apply,future_lapply)
+importFrom(future.batchtools,batchtools_slurm)
 importFrom(methods,is)
-importFrom(parallelly,availableWorkers)
+importFrom(missRanger,missRanger)
+importFrom(parsnip,boost_tree)
+importFrom(parsnip,linear_reg)
+importFrom(parsnip,mlp)
+importFrom(parsnip,set_engine)
+importFrom(parsnip,set_mode)
+importFrom(qs,qread)
+importFrom(qs,qsave)
+importFrom(recipes,recipe)
+importFrom(recipes,update_role)
+importFrom(rlang,as_name)
+importFrom(rlang,inject)
+importFrom(rlang,sym)
+importFrom(rsample,make_splits)
+importFrom(rsample,manual_rset)
+importFrom(rsample,vfold_cv)
 importFrom(sf,st_as_sf)
-importFrom(sf,st_as_sfc)
-importFrom(sf,st_bbox)
-importFrom(sf,st_buffer)
 importFrom(sf,st_coordinates)
-importFrom(sf,st_covered_by)
-importFrom(sf,st_crs)
-importFrom(sf,st_drop_geometry)
-importFrom(sf,st_geometry)
-importFrom(sf,st_sf)
-importFrom(sf,st_transform)
-importFrom(sf,st_union)
-importFrom(sf,st_within)
-importFrom(stars,read_stars)
-importFrom(stars,st_warp)
 importFrom(stats,predict)
-importFrom(stats,reshape)
-importFrom(terra,buffer)
-importFrom(terra,coltab)
-importFrom(terra,crop)
+importFrom(stats,sd)
+importFrom(stats,setNames)
+importFrom(stringi,stri_replace_all_regex)
+importFrom(targets,tar_resources)
+importFrom(targets,tar_resources_future)
 importFrom(terra,crs)
-importFrom(terra,describe)
-importFrom(terra,ext)
 importFrom(terra,extract)
-importFrom(terra,intersect)
-importFrom(terra,merge)
-importFrom(terra,mosaic)
-importFrom(terra,nlyr)
-importFrom(terra,project)
 importFrom(terra,rast)
-importFrom(terra,same.crs)
-importFrom(terra,sds)
-importFrom(terra,snap)
-importFrom(terra,tapp)
-importFrom(terra,vect)
-importFrom(utils,data)
-importFrom(utils,read.csv)
+importFrom(terra,subset)
+importFrom(terra,time)
+importFrom(terra,varnames)
+importFrom(tidyselect,all_of)
+importFrom(tigris,counties)
+importFrom(tune,tune_grid)
+importFrom(workflows,add_model)
+importFrom(workflows,add_recipe)
+importFrom(workflows,workflow)
+importFrom(yardstick,metric_set)
+importFrom(yardstick,rmse)