Skip to content

Commit

Permalink
Add gcp_gcs_list_md5s()
Browse files Browse the repository at this point in the history
wlandau-lilly committed Nov 9, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f89d25b commit 38d6e3c
Showing 2 changed files with 80 additions and 9 deletions.
52 changes: 43 additions & 9 deletions R/utils_gcp.R
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ gcp_gcs_head <- function(
bucket = gcp_gcs_bucket(),
version = NULL,
verbose = FALSE,
max_tries
max_tries = NULL
) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
@@ -39,7 +39,7 @@ gcp_gcs_exists <- function(
bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
version = NULL,
verbose = FALSE,
max_tries
max_tries = NULL
) {
!is.null(
gcp_gcs_head(
@@ -52,7 +52,12 @@ gcp_gcs_exists <- function(
)
}

gcp_gcs_bucket <- function(verbose = FALSE, max_tries) {
gcp_gcs_list_md5s <- function(
prefix,
bucket = gcp_gcs_bucket(),
verbose = TRUE,
max_tries = NULL
) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
@@ -62,9 +67,23 @@ gcp_gcs_bucket <- function(verbose = FALSE, max_tries) {
options(googleAuthR.tryAttempts = max_tries %|||% 5L)
)
gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
if_any(verbose, identity, suppressMessages) (
googleCloudStorageR::gcs_get_global_bucket()
if (verbose) {
tar_message_run(
"Listing objects in GCS bucket ",
bucket,
" prefix ",
prefix,
"..."
)
}
results <- googleCloudStorageR::gcs_list_objects(
prefix = prefix,
bucket = bucket,
detail = "full"
)
out <- as.list(results$md5)
names(out) <- results$name
out
}

gcp_gcs_download <- function(
@@ -73,7 +92,7 @@ gcp_gcs_download <- function(
bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
version = NULL,
verbose = FALSE,
max_tries
max_tries = NULL
) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
@@ -101,7 +120,7 @@ gcp_gcs_delete <- function(
bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
version = NULL,
verbose = FALSE,
max_tries
max_tries = NULL
) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
@@ -136,7 +155,7 @@ gcp_gcs_upload <- function(
metadata = list(),
predefined_acl = "private",
verbose = FALSE,
max_tries
max_tries = NULL
) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
@@ -165,7 +184,7 @@ gcp_gcs_upload <- function(
)
}

gcp_gcs_auth <- function(verbose = FALSE, max_tries) {
gcp_gcs_auth <- function(verbose = FALSE, max_tries = NULL) {
verbose <- verbose %|||% FALSE
if (isTRUE(tar_runtime$gcp_auth)) {
return()
@@ -188,4 +207,19 @@ gcp_gcs_auth <- function(verbose = FALSE, max_tries) {
tar_runtime$gcp_auth <- TRUE
invisible()
}

gcp_gcs_bucket <- function(verbose = FALSE, max_tries = NULL) {
verbose <- verbose %|||% FALSE
old_try_attempts <- getOption("googleAuthR.tryAttempts")
on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
if_any(
is.null(max_tries),
NULL,
options(googleAuthR.tryAttempts = max_tries %|||% 5L)
)
gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
if_any(verbose, identity, suppressMessages) (
googleCloudStorageR::gcs_get_global_bucket()
)
}
# nocov end
37 changes: 37 additions & 0 deletions tests/gcp/test-utils_gcp.R
Original file line number Diff line number Diff line change
@@ -325,3 +325,40 @@ tar_test("gcp_gcs_upload: upload twice, get the correct version", {
)
expect_equal(readLines(tmp), "second")
})

tar_test("gcp_gcs_list_md5s()", {
skip_if_no_gcp()
gcp_gcs_auth(max_tries = 5)
bucket <- random_bucket_name()
# needs to be a GCP project the tester auth has access to
project <- Sys.getenv("GCE_DEFAULT_PROJECT_ID")
googleCloudStorageR::gcs_create_bucket(
bucket,
projectId = project,
versioning = TRUE
)
on.exit(gcp_gcs_delete_bucket(bucket))
expect_equal(
gcp_gcs_list_md5s(prefix = "/", bucket = bucket),
list()
)
tmp <- tempfile()
writeLines("a", tmp)
for (key in c("w", "x", "y", "z")) {
gcp_gcs_upload(
file = tmp,
key = key,
bucket = bucket,
max_tries = 5
)
}
out <- gcp_gcs_list_md5s(prefix = "", bucket = bucket)
expect_equal(length(out), 4L)
expect_equal(sort(names(out)), sort(c("w", "x", "y", "z")))
for (etag in out) {
expect_true(is.character(etag))
expect_true(!anyNA(etag))
expect_equal(length(etag), 1L)
expect_gt(nchar(etag), 10L)
}
})

0 comments on commit 38d6e3c

Please sign in to comment.