Merge branch 'develop' into feat/nf-datasets

R/schema_utils.R

@@ -20,6 +20,8 @@ as_table_schema <- function(df,
                             schema, 
                             list_truncate = FALSE) {
 
+  .check_login()
+  if("data.table" %in% class(df)) df <- as.data.frame(df)
   if("synapseclient.table.Schema" %in% class(schema) && reticulate::py_has_attr(schema, "columns_to_store")) { 
     col_schema <- schema$columns_to_store
   } else {
@@ -40,8 +42,9 @@ as_table_schema <- function(df,
     values <- df[[i]]
     if(grepl("STRING", col_type[i])) {
       maxsize <-  col_schema[[i]]$maximumSize
+      if(anyNA(values)) stop("Please remove NA values from STRING column", names(df)[i])
       size_fail <- sapply(values, function(x) any(nchar(x) > maxsize))
-      if(any(size_fail)) stop(paste("Characters in", names(df)[i], "exceeds max size of", maxlen))
+      if(any(size_fail)) stop(paste("Characters in", names(df)[i], "exceeds max size of", maxsize))
     }
     if(grepl("*_LIST", col_type[i])) {
       maxlen <- col_schema[[i]]$maximumListLength

tests/testthat/test_synapse_schema_utils.R

@@ -0,0 +1,66 @@
+# Test fixture is table syn49378540 on Synapse, with schema that includes INTEGER and STRING and STRING_LIST columns with character size and character + list size limits.
+# These tests mainly focus on the character and list validation and JSON encoding functionality that `as_table_schema` provides
+# in order to successfully append data conforming to an existing Synapse schema
+
+test_that("`as_table_schema` works as expected for data that can be fit into schema without issue", {
+  skip_if_no_synapseclient()
+  skip_if_no_pandas()
+  skip_if_no_login()
+  test_fixture <- "syn49378540"
+  test_data <- data.frame(Movie = "The Sound of Music",
+                          Year = 1965,
+                          Favorites = I(list(list("raindrops","whiskers", "kettles"))))
+  test_data_storable <- as_table_schema(test_data, schema = test_fixture)
+  testthat::expect_s3_class(.syn$store(test_data_storable), "synapseclient.table.CsvFileTable")
+})
+
+test_that("`as_table_schema` errors for data with missing column", {
+  skip_if_no_synapseclient()
+  skip_if_no_pandas()
+  skip_if_no_login()
+  test_fixture <- "syn49378540"
+  test_data <- data.frame(Movie = "The Sound of Music",
+                          Year = 1965)
+  testthat::expect_error(as_table_schema(test_data, schema = test_fixture))
+
+})
+
+
+test_that("`as_table_schema` errors for data that exceeds list length as specified in schema for LIST column and truncation is not allowed", {
+  skip_if_no_synapseclient()
+  skip_if_no_pandas()
+  skip_if_no_login()
+  test_fixture <- "syn49378540"
+  test_data <- data.frame(Movie = "The Sound of Music",
+                          Year = 1965,
+                          Favorites = I(list(list("raindrops", "whiskers", "kettles", "mittens")))) # exceeds list length of 3
+  testthat::expect_error(as_table_schema(test_data, schema = test_fixture, list_truncate = FALSE))
+
+})
+
+
+test_that("`as_table_schema` returns result with warning for data that exceeds list length specified in schema for LIST column but truncated to fit", {
+  skip_if_no_synapseclient()
+  skip_if_no_pandas()
+  skip_if_no_login()
+  test_fixture <- "syn49378540"
+  test_data <- data.frame(Movie = "The Sound of Music",
+                          Year = 1965,
+                          Favorites = I(list(list("raindrops", "whiskers", "kettles", "mittens")))) # exceeds list length of 3
+  testthat::expect_warning(as_table_schema(test_data, schema = test_fixture, list_truncate = TRUE))
+
+})
+
+
+test_that("`as_table_schema` errors for data that exceeds character limits specified in schema for STRING_LIST column", {
+  skip_if_no_synapseclient()
+  skip_if_no_pandas()
+  skip_if_no_login()
+  test_fixture <- "syn49378540"
+  test_data <- data.frame(Movie = "The Sound of Music",
+                          Year = 1965,
+                          Favorites = I(list(list("raindrops on roses", "whiskers")))) # exceeds character size of 15 for first value
+  testthat::expect_error(as_table_schema(test_data, schema = test_fixture))
+
+})
+