Skip to content

Commit

Permalink
Merge pull request #32 from amashadihossein/dev
Browse files Browse the repository at this point in the history 
Add pinsLabkey
  • Loading branch information
@camorosi
camorosi authored Apr 22, 2024
2 parents 55d6b44 + 96484e6 commit 1f339af
Show file tree
Hide file tree
Showing 5 changed files with 161 additions and 113 deletions.
13 changes: 8 additions & 5 deletions DESCRIPTION
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Package: dpdeploy
Title: A package to manage deployment of data products
Version: 0.1.0
Version: 0.2.0
Authors@R: c(
person("Afshin", "Mashadi-Hossein", , "[email protected]", role = c("aut", "cre")),
person("Yue", "Jiang", , "[email protected]", role = "aut")
person(given = "Afshin", family = "Mashadi-Hossein", email ="[email protected]", role = c("aut", "cre")),
person(given = "Yue", family = "Jiang", email = "[email protected]", role = "aut")
)
Description: An R package to manage deployment of data product to remote.
License: GPL-3
Expand All @@ -12,8 +12,8 @@ Imports:
aws.s3,
aws.signature,
cli,
dpbuild (>= 0.1.0),
dpi (>= 0.1.0),
dpbuild (>= 0.2.0),
dpi (>= 0.2.0),
dplyr,
ellipsis,
fs,
Expand All @@ -22,6 +22,7 @@ Imports:
lifecycle,
magrittr,
pins (>= 1.2.0),
pinsLabkey,
purrr,
rlang (>= 0.4.11),
withr,
Expand All @@ -33,3 +34,5 @@ Encoding: UTF-8
LazyData: true
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.2.3
Remotes:
github::camorosi/pinsLabkey
4 changes: 4 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# dpdeploy 0.2.0

* Add back support for LabKey boards (#29). `pinsLabkey` is now required to work with LabKey boards

# dpdeploy 0.1.0

## Breaking changes
Expand Down
55 changes: 39 additions & 16 deletions R/dp_deploy.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,6 @@ dp_deployCore.s3_board <- function(conf, project_path, d, dlog, git_info,
versioned = T
)

# This is force data.txt sync prior to pinning to address pins bug where
# versions can be lost
# ver_current <- pins::pin_versions(
# name = as.character(attr(d, "dp_name")),
# board = conf$board_params$board_alias
# )

pins::pin_write(
x = d,
name = as.character(attr(d, "dp_name")),
Expand All @@ -97,6 +90,41 @@ dp_deployCore.s3_board <- function(conf, project_path, d, dlog, git_info,
return(TRUE)
}

#' @keywords internal
dp_deployCore.labkey_board <- function(conf, project_path, d, dlog, git_info,
verbose = F, ...) {
if (verbose) {
print(glue::glue("Deploying to LabKey remote"))
}

# define board and pin dp to LabKey
labkey_creds <- conf$creds
if (labkey_creds$api_key == "") {
stop("Please check LabKey credentials. You need to provide api_key")
}

board <- pinsLabkey::board_labkey(
board_alias = conf$board_params$board_alias,
api_key = labkey_creds$api_key,
base_url = conf$board_params$url,
folder = conf$board_params$folder,
versioned = T,
subdir = "daap/"
)

pinsLabkey::pin_write(
x = d,
name = as.character(attr(d, "dp_name")),
board = board,
description = as.character(attr(d, "branch_description"))
)

# Update dpboard_log
dpboardlog_update(conf = conf, dlog = dlog, git_info = git_info)

return(TRUE)
}


#' @keywords internal
dp_deployCore.local_board <- function(conf, project_path, d, dlog, git_info,
Expand All @@ -107,15 +135,10 @@ dp_deployCore.local_board <- function(conf, project_path, d, dlog, git_info,


# define board and pin dp to local board
board_object <- pins::board_folder(path = file.path(conf$board_params$folder, "daap"),
versioned = T)

# This is force data.txt sync prior to pinning to address pins bug where
# versions can be lost. not sure necessary for local board but not harmful
# ver_current <- pins::pin_versions(
# name = as.character(attr(d, "dp_name")),
# board = conf$board_params$board_alias
# )
board_object <- pins::board_folder(
path = file.path(conf$board_params$folder, "daap"),
versioned = T
)

pins::pin_write(
x = d,
Expand Down
112 changes: 61 additions & 51 deletions R/dpinput_sync.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ dpinput_sync <- function(conf, input_map, verbose = F, ...) {

synced_map <- sync_iterate(
input_map = input_map,
board_object = board, #get_inputboard_alias(conf),
board_object = board,
skip_sync = skip_sync,
rewrite_ok = rewrite_ok,
verbose = verbose
Expand Down Expand Up @@ -102,34 +102,42 @@ init_board.s3_board <- function(conf) {
aws.signature::locate_credentials(profile = aws_creds$profile_name)$secret
}

input_params <- list(
bucket_name = conf$board_params$bucket_name,
region = conf$board_params$region,
aws_key = aws_creds$key,
aws_secret = aws_creds$secret
)

pins::board_s3(
prefix = "dpinput/",
bucket = input_params$bucket_name,
region = input_params$region,
access_key = input_params$aws_key,
secret_access_key = input_params$aws_secret,
bucket = conf$board_params$bucket_name,
region = conf$board_params$region,
access_key = aws_creds$key,
secret_access_key = aws_creds$secret,
versioned = TRUE
)
}

#' @keywords internal
init_board.labkey_board <- function(conf) {
# define board and pin dp to LabKey
labkey_creds <- conf$creds
if (labkey_creds$api_key == "") {
stop("Please check LabKey credentials. You need to provide api_key")
}

pinsLabkey::board_labkey(
board_alias = conf$board_params$board_alias,
api_key = labkey_creds$api_key,
base_url = conf$board_params$url,
folder = conf$board_params$folder,
versioned = T,
subdir = "dpinput/"
)
}

#' @keywords internal
init_board.local_board <- function(conf) {
pins::board_folder(path = file.path(conf$board_params$folder, "dpinput"),
versioned = T)
pins::board_folder(
path = file.path(conf$board_params$folder, "dpinput"),
versioned = T
)
}

# #' @keywords internal
# get_inputboard_alias <- function(conf) {
# inputboard_alias <- paste0(conf$board_params$board_alias, "_dpinput")
# return(inputboard_alias)
# }

#' @keywords internal
to_description <- function(input_i) {
Expand Down Expand Up @@ -189,17 +197,28 @@ pathnames_reroot <- function(pathnames, new_root = "input_files") {
sync_iterate <- function(input_map, board_object, skip_sync, rewrite_ok = F,
verbose) {
synced_map <- purrr::map(.x = input_map, .f = function(input_i) {

pin_name_exists <- pins::pin_exists(board = board_object, name = input_i$metadata$name)
if (board_object$board == "pins_board_labkey") {
pin_name_exists <- pinsLabkey::pin_exists(board = board_object, name = input_i$metadata$name)
} else {
pin_name_exists <- pins::pin_exists(board = board_object, name = input_i$metadata$name)
}

if (pin_name_exists) {
synced_versions <- pins::pin_versions(
name = input_i$metadata$name,
board = board_object
) %>%
dplyr::pull(hash)
if (board_object$board == "pins_board_labkey") {
synced_versions <- pinsLabkey::pin_versions(
name = input_i$metadata$name,
board = board_object
) %>%
dplyr::pull(hash)
} else {
synced_versions <- pins::pin_versions(
name = input_i$metadata$name,
board = board_object
) %>%
dplyr::pull(hash)
}

input_i$metadata$synced <- input_i$metadata$pin_version %in% synced_versions
input_i$metadata$synced <- input_i$metadata$pin_version %in% synced_versions
} else {
input_i$metadata$synced <- F
}
Expand All @@ -220,12 +239,21 @@ sync_iterate <- function(input_map, board_object, skip_sync, rewrite_ok = F,
}

if (!skip_pin_to_remote) {
tmp_pind <- try(pins::pin_write(
x = input_i$data,
name = input_i$metadata$name,
board = board_object,
description = input_i$metadata$description
))
if (board_object$board == "pins_board_labkey") {
tmp_pind <- try(pinsLabkey::pin_write(
x = input_i$data,
name = input_i$metadata$name,
board = board_object,
description = input_i$metadata$description
))
} else {
tmp_pind <- try(pins::pin_write(
x = input_i$data,
name = input_i$metadata$name,
board = board_object,
description = input_i$metadata$description
))
}

input_i$metadata$synced <- TRUE
sync_attempt_state <- "completed"
Expand All @@ -236,24 +264,6 @@ sync_iterate <- function(input_map, board_object, skip_sync, rewrite_ok = F,
sync_alrt <- cli::cli_alert_warning
}

# get_remote_pin_version <- pins::pin_versions(
# name = input_i$metadata$name,
# board = board_object
# ) %>%
# dplyr::arrange(dplyr::desc(version)) %>%
# dplyr::filter(dplyr::row_number()==1) %>%
# dplyr::pull(hash)
#
# input_i$metadata$description <- to_description(input_i = input_i)
#
# if (length(get_remote_pin_version) > 1) {
# latest_pin_version <- sort(get_remote_pin_version)[length(get_remote_pin_version)]
# } else {
# latest_pin_version <- get_remote_pin_version
# }
#
# input_i$metadata$pin_version <- latest_pin_version

if (verbose) {
sync_alrt(glue::glue(
"Input {input_i$metadata$name}, version ",
Expand Down
Loading

0 comments on commit 1f339af

Please sign in to comment.