Skip to content

Commit

Permalink
Merge pull request #598 from Sage-Bionetworks/develop
Browse files Browse the repository at this point in the history 
v24.5.1
  • Loading branch information
@afwillia
afwillia authored May 23, 2024
2 parents 2a75726 + acc8e63 commit 0402ba5
Show file tree
Hide file tree
Showing 40 changed files with 1,035 additions and 423 deletions.
2 changes: 2 additions & 0 deletions .Rbuildignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
^\.pre-commit-config\.yaml$
^renv$
^renv\.lock$
^.venv
^schematic$
14 changes: 13 additions & 1 deletion .github/workflows/docker_build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,16 @@ jobs:
build-args: |
DCA_VERSION=${{ env.DCA_VERSION }}
- name: Lowercase image name for trivy
id: string
uses: ASzc/change-string-case-action@v6
with:
string: ${{ env.IMAGE_PATH }}

- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: '${{ steps.string.outputs.lowercase }}:${{ steps.meta.outputs.version }}'
format: 'table'
ignore-unfixed: true
severity: 'CRITICAL,HIGH'
2 changes: 1 addition & 1 deletion DESCRIPTION
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Authors: Rongrong Chai, Xengie Doan, Milen Nikolov, Sujay Patil, Robert Allaway,
License: file LICENSE
Encoding: UTF-8
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.2.3
RoxygenNote: 7.3.1
Imports: httr, dplyr, jsonlite, shinyjs, yaml, promises, readr
Suggests:
covr
189 changes: 118 additions & 71 deletions R/schematic_rest_api.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,29 @@ check_success <- function(x){
#' @param url URI of API endpoint
#' @param access_token Synapse PAT
#' @param asset_view ID of view listing all project data assets
#' @param dataset_id the parent ID of the manifest
#' @param manifest_id the parent ID of the manifest
#' @param as_json if True return the manifest in JSON format
#' @returns a csv of the manifest
#' @export
manifest_download <- function(url = "http://localhost:3001/v1/manifest/download", access_token, asset_view, dataset_id, as_json=TRUE, new_manifest_name=NULL) {
request <- httr::GET(
url = url,
httr::add_headers(Authorization = sprintf("Bearer %s", access_token)),
query = list(
asset_view = asset_view,
dataset_id = dataset_id,
manifest_download <- function(url = "http://localhost:3001/v1/manifest/download", access_token, manifest_id, as_json=TRUE, new_manifest_name=NULL) {

req <- httr2::request(url) |>
httr2::req_retry(
max_tries = 3,
is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503, 403)
) |>
httr2::req_error(is_error = \(r) FALSE)
resp <- req |>
httr2::req_headers(Authorization = sprintf("Bearer %s", access_token)) |>
httr2::req_url_query(
manifest_id = manifest_id,
as_json = as_json,
new_manifest_name = new_manifest_name
)
)

check_success(request)
response <- httr::content(request, type = "application/json")

# Output can have many NULL values which get dropped or cause errors. Set them to NA
nullToNA <- function(x) {
x[sapply(x, is.null)] <- NA
return(x)
}
df <- do.call(rbind, lapply(response, rbind))
nullToNA(df)

) |>
httr2::req_perform()
resp |> httr2::resp_body_string() |>
(function(d) gsub('NaN', '"NA"', x = d))() |>
jsonlite::fromJSON()
}

#' schematic rest api to generate manifest
Expand Down Expand Up @@ -132,6 +128,7 @@ manifest_validate <- function(url="http://localhost:3001/v1/model/validate",
project_scope = NULL,
access_token,
asset_view = NULL,
json_str = NULL,
data_model_labels = "class_label") {

flattenbody <- function(x) {
Expand All @@ -153,35 +150,68 @@ manifest_validate <- function(url="http://localhost:3001/v1/model/validate",
}, names(x), x, USE.NAMES = FALSE, SIMPLIFY = FALSE))
}

req <- httr::POST(url,
httr::add_headers(Authorization = sprintf("Bearer %s", access_token)),
query=flattenbody(list(
schema_url=schema_url,
data_type=data_type,
restrict_rules=restrict_rules,
project_scope = project_scope,
asset_view = asset_view,
data_model_labels = data_model_labels)),
body=list(file_name=httr::upload_file(file_name))
)
if (all(is.null(json_str), is.null(file_name))) {
stop("Must provide either a file to upload or a json")
}

# Format server error in a way validationResult can handle
if (httr::http_status(req)$category == "Server error") {
return(
list(
list(
"errors" = list(
Row = NA, Column = NA, Value = NA,
Error = sprintf("Cannot validate manifest: %s",
httr::http_status(req)$message)
)
)
)
)
if (is.null(json_str)) {
reqs <- httr2::request(url) |>
httr2::req_retry(
max_tries = 3,
is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503, 504, 403)
) |>
httr2::req_throttle(1/2) |>
httr2::req_error(is_error = \(reqs) FALSE)
resp <- reqs |>
httr2::req_headers(Authorization = sprintf("Bearer %s", access_token)) |>
httr2::req_url_query(
schema_url=schema_url,
data_type=data_type,
restrict_rules=restrict_rules,
project_scope = project_scope,
data_model_labels = data_model_labels,
asset_view = asset_view
) |>
httr2::req_body_multipart(file_name=curl::form_file(file_name)) |>
httr2::req_perform()
} else {
req <- httr2::request(url) |>
httr2::req_throttle(1)
resp <- req |>
httr2::req_headers(Authorization = sprintf("Bearer %s", access_token)) |>
httr2::req_url_query(
schema_url=schema_url,
data_type=data_type,
restrict_rules=restrict_rules,
project_scope = project_scope,
asset_view = asset_view,
data_model_labels = data_model_labels,
json_str = json_str
) |>
#httr2::req_retry(
# max_tries = 3,
# is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503, 504)
#) |>
#httr2::req_error(is_error = \(resp) FALSE) |>
httr2::req_perform()
}
check_success(req)
annotation_status <- httr::content(req)
annotation_status

# Format server error in a way validationResult can handle
# if (httr2::resp_is_error(resp)) {
# return(
# list(
# list(
# "errors" = list(
# Row = NA, Column = NA, Value = NA,
# Error = sprintf("Cannot validate manifest: %s",
# httr2::resp_status_desc(resp)
# )
# )
# )
# )
# )
# }
httr2::resp_body_json(resp)
}


Expand All @@ -191,8 +221,18 @@ manifest_validate <- function(url="http://localhost:3001/v1/model/validate",
#' @param schema_url URL to a schema jsonld
#' @param data_type Type of dataset
#' @param dataset_id Synapse ID of existing manifest
#' @param access_token Synapse login cookie, PAT, or API key.
#' @param csv_file Filepath of csv to validate
#' @param restrict_rules Default = FALSE
#' @param access_token Synapse login cookie, PAT, or API key
#' @param json_str Json string to submit
#' @param asset_view Synapse fileview
#' @param manifest_record_type Default = "table_and_file"
#' @param file_name Name of file
#' @param table_manipulation Default = "replace"
#' @param hide_blanks Default = FALSE
#' @param table_column_names Default = "class_and_label"
#' @param annotation_keys Default = "class_and_label"
#' @param data_model_labels Default = "class_and_label"
#' @param upload_file_annotations Default = TRUE
#'
#' @returns TRUE if successful upload or validate errors if not.
#' @export
Expand All @@ -210,7 +250,8 @@ model_submit <- function(url="http://localhost:3001/v1/model/submit",
hide_blanks=FALSE,
table_column_names="class_label",
annotation_keys="class_label",
data_model_labels="class_label") {
data_model_labels="class_label",
file_annotations_upload=TRUE) {
req <- httr::POST(url,
httr::add_headers(Authorization = sprintf("Bearer %s", access_token)),
query=list(
Expand All @@ -225,7 +266,8 @@ model_submit <- function(url="http://localhost:3001/v1/model/submit",
table_column_names=table_column_names,
annotation_keys=annotation_keys,
data_model_labels=data_model_labels,
hide_blanks=hide_blanks),
hide_blanks=hide_blanks,
file_annotations_upload=file_annotations_upload),
body=list(file_name=httr::upload_file(file_name))
#body=list(file_name=file_name)
)
Expand All @@ -249,23 +291,28 @@ model_component_requirements <- function(url="http://localhost:3001/v1/model/com
as_graph = FALSE,
data_model_labels = "class_label") {

req <- httr::GET(url,
query = list(
schema_url = schema_url,
source_component = source_component,
as_graph = as_graph,
data_model_labels = data_model_labels
))

check_success(req)
cont <- httr::content(req)

if (inherits(cont, "xml_document")){
err_msg <- xml2::xml_text(xml2::xml_child(cont, "head/title"))
stop(sprintf("%s", err_msg))
reqs <- httr2::request(url) |>
httr2::req_retry(
max_tries = 5,
is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503)
) |>
httr2::req_error(is_error = \(r) FALSE)
resp <- reqs |>
httr2::req_url_query(
schema_url = schema_url,
source_component = source_component,
data_model_labels = data_model_labels,
as_graph = as_graph
) |>
#httr2::req_retry(max_tries = 3) |>
httr2::req_perform()
if (httr2::resp_is_error(resp)) {
warning(sprintf("model/component-requirement failed for %s. returning empty list. %s",
source_component, httr2::resp_body_json(resp)$title))
return(list())
}

cont
resp |>
httr2::resp_body_json()

}

Expand All @@ -290,7 +337,7 @@ storage_project_datasets <- function(url="http://localhost:3001/v1/storage/proje
asset_view=asset_view,
project_id=project_id)
)

check_success(req)
httr::content(req)
}
Expand Down Expand Up @@ -364,7 +411,7 @@ get_asset_view_table <- function(url="http://localhost:3001/v1/storage/assets/ta
if (return_type=="json") {
return(list2DF(fromJSON(httr::content(req))))
} else {
csv <- readr::read_csv(httr::content(req))
csv <- readr::read_csv(httr::content(req), show_col_types = FALSE)
return(csv)
}

Expand Down
66 changes: 49 additions & 17 deletions R/synapse_rest_api.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,19 @@ synapse_is_certified <- function(url="https://repo-prod.prod.sagebase.org/repo/v
#' @param auth Synapse PAT
#'
#' @export
synapse_get <- function(url = "https://repo-prod.prod.sagebase.org/repo/v1/entity/",
synapse_get <- function(url = "https://repo-prod.prod.sagebase.org/repo/v1/entity",
id, auth) {

if (is.null(id)) stop("id cannot be NULL")
req_url <- file.path(url, id)
req <- httr::GET(req_url,
httr::add_headers(Authorization=paste0("Bearer ", auth)))

# Send error if unsuccessful query
status <- httr::http_status(req)
if (status$category != "Success") stop(status$message)

cont <- httr::content(req)
dplyr::bind_rows(cont)

req <- httr2::request(file.path(url, id))
resp <- req |>
httr2::req_retry(
max_tries = 5,
is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503, 403)
) |>
httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |>
httr2::req_perform()
resp |> httr2::resp_body_json()
}


Expand Down Expand Up @@ -216,9 +214,16 @@ synapse_table_query <- function(id, auth, query, partMask=0x7F) {
#' @param auth Synapse token
synapse_table_get <- function(id, async_token, auth) {
url <- file.path("https://repo-prod.prod.sagebase.org/repo/v1/entity", id,"table/query/async/get", async_token)
req <- httr::GET(url = url,
httr::add_headers(Authorization=paste0("Bearer ", auth)))
httr::content(req)
request <- httr2::request(url)
response <- request |>
httr2::req_retry(
max_tries = 5,
is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503, 202, 403)
) |>
httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |>
httr2::req_perform()
httr2::resp_body_json(response)

}

#' @title Get column names from a Synapse table
Expand All @@ -245,7 +250,6 @@ synapse_storage_projects <- function(id, auth, select_cols = c("id", "name", "pa
select_cols_format <- paste(select_cols, collapse = ", ")
query <- sprintf("select distinct %s from %s", select_cols_format, id)
request <- synapse_table_query(id, auth, query, partMask = 0x1)
Sys.sleep(1)
response <- synapse_table_get(id, request$token, auth)

setNames(
Expand Down Expand Up @@ -278,6 +282,34 @@ synapse_download_file_handle <- function(dataFileHandleId, id, auth, filepath=NU
download_url <- httr::content(request)
destfile <- ifelse(is.null(filepath), tempfile(), filepath)
download.file(download_url, destfile)
if (is.null(filepath)) readr::read_csv(destfile)
if (is.null(filepath)) readr::read_csv(destfile, show_col_types = FALSE)

}

#' @title Download the storage manifest records from an asset view table
synapse_get_manifests_in_asset_view <- function(id, auth) {
request <- synapse_table_query(
id = id,
auth = auth,
query = paste("select * from",
id,
"where name like 'synapse|_storage|_manifest|_%' escape '|'"),
partMask = 0x11)
response <- synapse_table_get(
id = id,
async_token = request$token,
auth = auth)
# Format the query results by reshaping the results list and getting column
# names. partMask 0x11 gets queryResults and column names
setNames(
tibble::as_tibble(
t(
vapply(
response$queryResult$queryResults$rows, function(x) {
null_ind <- which(sapply(x$values, is.null))
x$values[null_ind] <- NA
unlist(x$values)
},
character(length(response$columnModels))))),
vapply(response$columnModels, function(x) x$name,character(1L)))
}
Loading

0 comments on commit 0402ba5

Please sign in to comment.