Merge pull request #201 from natverse/feature/more-expanders

Direct support for flyem shorturls including via tinyurl

Author: jefferis

R/brainmaps-api.R

@@ -506,11 +506,11 @@ read.neurons.brainmaps<-function(x, OmitFailures=NA, df=NULL, ... ) {
 #'
 #' @return A list containing the following fields \itemize{
 #'
-#'   \item{\code{nvertices}}{ The number of vertices (n)}
+#'   \item \code{nvertices} The number of vertices (n)
 #'
-#'   \item{\code{nedges}}{ The number of edges (m)}
+#'   \item \code{nedges} The number of edges (m)
 #'
-#'   \item{\code{vertices}}{ A \code{n} x 3 matrix of vertex locations}
+#'   \item \code{vertices} A \code{n} x 3 matrix of vertex locations
 #'
 #'   }
 #' @export

R/flywire-api.R

@@ -26,19 +26,19 @@
 #'   or error. The default value (\code{TRUE}) will skip over errors, while
 #'   \code{NA}) will result in a hard stop on error. See \code{\link{nlapply}}
 #'   for more details.
-#' @return A data frame with values itemize{
+#' @return A data frame with values \itemize{
 #'
-#'   \item{operation_id}{ a unique id for the edit}
+#'   \item \code{operation_id} a unique id for the edit
 #'
-#'   \item{timestamp}{ in POSIXct format, to the nearest ms}
+#'   \item \code{timestamp} in POSIXct format, to the nearest ms
 #'
-#'   \item{user_id}{ numeric id for the user responsible for the edit}
+#'   \item \code{user_id} numeric id for the user responsible for the edit
 #'
-#'   \item{is_merge}{ whether it was a merge or a split}
+#'   \item \code{is_merge} whether it was a merge or a split
 #'
-#'   \item{user_name}{ as a string}
+#'   \item \code{user_name} as a string
 #'
-#'   \item{before_root_ids and after_root_ids}{ as space separated strings}
+#'   \item \code{before_root_ids and after_root_ids} as space separated strings
 #'
 #'   }
 #'

R/flywire-fetch.R

@@ -16,7 +16,8 @@
 #' @inheritParams brainmaps_fetch
 #' @param return One of "parsed", "text" (for raw JSON), or "response"
 #' @param token Optional chunkedgraph token (otherwise the default one for the
-#'   current segmentation will be used).
+#'   current segmentation will be used). Use \code{NA} to suppress use of a
+#'   token.
 #' @param config (optional) curl options, see \code{httr::\link[httr]{config}}
 #'   for details.
 #'
@@ -51,7 +52,8 @@ flywire_fetch <- function(url,
     config = httr::config()
   if(is.null(token))
     token = chunkedgraph_token()
-  config = c(config, add_headers(Authorization = paste("Bearer", token)))
+  if(!isTRUE(is.na(token)))
+    config = c(config, add_headers(Authorization = paste("Bearer", token)))
 
   #Step 3: choose the actual request function to use, if cache on try the memoised one
   # otherwise use the retry from httr..

R/flywire-urls.R

@@ -76,12 +76,22 @@ flywire_shortenurl <- function(x, include_base=TRUE, baseurl=NULL, cache=TRUE, .
 
 
 #' @description \code{flywire_expandurl} expands shortened URLs into a full
-#'   neuroglancer JSON scene specification. If the active segmentation
+#'   neuroglancer JSON scene specification. If the link references a specific
+#'   version of neuroglancer on a specific host URL then that will be used as
+#'   the base of the expanded URL. This is nearly always the case, but should
+#'   this ever not be so, then if the active segmentation
 #'   (\code{\link{choose_segmentation}}) is a flywire segmentation then that is
-#'   used to define the initial part of the output URL, otherwise the
+#'   used to define the initial part of the output URL. Failing this, the
 #'   \code{flywire31} segmentation is used.
 #'
-#'   \code{flywire_expandurl} will also expand tinyurl.com URLs.
+#'   \code{flywire_expandurl} will also expand tinyurl.com URLs as well as those
+#'   referencing a json fragment on a google cloud bucket (such as the flyem
+#'   link shortener). If a tinyurl.com URL maps to a short URL referencing a
+#'   json fragment, then they will successively be expanded.
+#'
+#'   Finally, if the URL is actually already expanded, then this will be
+#'   returned unmodified. This is a change in behaviour as of May 2024
+#'   (previously an error was thrown).
 #' @param json.only Only return the JSON fragment rather than the neuroglancer
 #'   URL
 #' @export
@@ -91,31 +101,47 @@ flywire_shortenurl <- function(x, include_base=TRUE, baseurl=NULL, cache=TRUE, .
 #' flywire_expandurl("https://globalv1.flywire-daf.com/nglstate/5747205470158848")
 #' flywire_expandurl("https://tinyurl.com/rmr58jpn")
 #' }
+#' \dontrun{
+#' flywire_expandurl("https://tinyurl.com/flywirehb2")
+#' }
 #' @rdname flywire_shortenurl
 flywire_expandurl <- function(x, json.only=FALSE, cache=TRUE, ...) {
   checkmate::assert_character(x, pattern="^http[s]{0,1}://")
   if(length(x)>1) {
     res=pbapply::pbsapply(x, flywire_expandurl, json.only=json.only, cache=cache, ...)
     return(res)
   }
+  url=x
   if(grepl("tinyurl.com", x, fixed = TRUE)) {
     # head should redirect to expanded URL
-    x=httr::HEAD(x)$url
-    if(json.only)
-      x=ngl_decode_scene(x, return.json = TRUE)
-    return(x)
+    url=httr::HEAD(x, config(followlocation=TRUE))$url
+    # occasionally we seem to get this ... have to GET
+    if(grepl("comsync.lijit.com", url, fixed = T))
+      url=httr::GET(url, config(followlocation=TRUE))$url
+    x=url
   }
 
-  if(isFALSE(su <- shorturl(x)))
-    stop("This doesn't look like a shortened neuroglancer URL: ", x)
-  x=flywire_fetch(su, cache=cache, return='text', ...)
+  if(isFALSE(su <- shorturl(url))) {
+    if(json.only) return(ngl_decode_scene(x, return.json = TRUE))
+    else return(x)
+  }
+  # suppress use of token (with NA) if we are not talking to a CAVE link server
+  stateserverurl=isTRUE(grepl("nglstate(/api/v[0-9])*/[0-9]+$", su))
+  use_token=if(stateserverurl) NULL else NA
+  x=flywire_fetch(su, cache=cache, return='text', token=use_token, ...)
 
   if(isFALSE(json.only)) {
-    # if we have a flywire segmentation active use that to encode URL
-    flywire_active=isTRUE(grepl('flywire.ai', getOption('fafbseg.sampleurl')))
-    x <- if (flywire_active)
+    baseurl=try({
+      pu=httr::parse_url(url)
+      pu$fragment=NULL
+      httr::build_url(pu)
+    })
+    x <- if(!inherits(baseurl, 'try-error')) {
+      ngl_encode_url(x, baseurl = baseurl)
+    } else if(flywire_active <- isTRUE(grepl('flywire.ai', getOption('fafbseg.sampleurl')))) {
+      # if we have a flywire segmentation active use that to encode URL
       ngl_encode_url(x)
-    else
+    } else
       with_segmentation('flywire31', ngl_encode_url(x))
   }
   x

R/merge-groups.R

@@ -9,10 +9,10 @@
 #' @return vector of segment ids (in ascending order) or when
 #'   \code{return.groups=TRUE} a \code{data.frame} with columns \itemize{
 #'
-#'   \item{segment}{ the integer segment id, as a numeric (double) column}
+#'   \item \code{segment} the integer segment id, as a numeric (double) column
 #'
-#'   \item{group}{ an arbitrary group id starting from 1 OR the canonical
-#'   segment id (see details), an integer or numeric (double), respectively}
+#'   \item \code{group} an arbitrary group id starting from 1 OR the canonical
+#'   segment id (see details), an integer or numeric (double), respectively
 #'
 #'   }
 #' @details segment ids in \code{ffn16reseg-ms1000_md0.02_c0.6_iou0.7} always

R/read_merge_info.R

@@ -14,9 +14,9 @@
 #'
 #' @return a \code{data.frame} with columns \itemize{
 #'
-#'   \item{id1,id2}{ Segment ids to be merged}
+#'   \item \code{id1,id2} Segment ids to be merged
 #'
-#'   \item{x,y,z}{ Location (in nm) of merge point}
+#'   \item \code{x,y,z} Location (in nm) of merge point
 #'
 #'   }
 #' @export

R/urls.R

@@ -77,7 +77,7 @@ ngl_decode_scene <- function(x, return.json=FALSE, simplifyVector = TRUE,
         # This looks like a URL
         # special case, expand shortened flywire URLs
         if (!isFALSE(su <- shorturl(x))) {
-          saved_url = flywire_expandurl(su, json.only = FALSE, ...)
+          saved_url = flywire_expandurl(x, json.only = FALSE, ...)
           x <- ngl_decode_scene(saved_url, return.json = T)
         } else {
           saved_url <- x
@@ -124,7 +124,20 @@ shorturl <- function(x) {
   if(px$hostname %in% c("tinyurl.com"))
     return(x)
   # looks like fully expanded fragment
-  if(!is.null(px$fragment)) return(FALSE)
+  if(!is.null(px$fragment)) {
+    url <- px$fragment
+    # remove middleauth prefix - we'll be using flywire_fetch to get the URL
+    if(isTRUE(substr(url, 1, 12) == "!middleauth+"))
+      url=paste0("!", substr(url,13,nchar(url)))
+
+    if(isTRUE(substr(url, 1, 6) == "!gs://")) {
+      path = substr(url, 6, nchar(url))
+      gu = "https://storage.googleapis.com"
+      return(paste0(gu, path))
+    } else if(isTRUE(substr(url, 1, 9) == "!https://")) {
+      return(substr(url, 2, nchar(url)))
+    } else return(FALSE)
+  }
   if(!is.null(px$query$json_url))
     return(px$query$json_url)
   # may have been a bare URL, but in that case check path

man/brainmaps_skeleton.Rd

@@ -99,15 +99,19 @@ test_that("can expand a flywire url to get segments", {
       "720575940637384518"
     )
   )
+  # check long url comes back unaltered
+  fsu=fafbseg::choose_segmentation('flywire31', set = F)$fafbseg.sampleurl
+  expect_equal(flywire_expandurl(fsu), fsu)
 
-  expect_error(
-    flywire_expandurl(
-      fafbseg::choose_segmentation('flywire31', set = F)$fafbseg.sampleurl
-    ),
-    'shortened neuroglancer'
-  )
   expect_known_hash(flywire_expandurl('https://tinyurl.com/rmr58jpn'),
                     hash = 'a5fb89f6f9')
+
+  # make sure we can expand a recursive tinyurl
+  expect_equal(flywire_expandurl("https://tinyurl.com/flywirehb2"),
+                   flywire_expandurl("https://neuroglancer-demo.appspot.com/#!gs://flyem-user-links/short/2023-08-26.151006.json"))
+
+  expect_is(flywire_expandurl("https://spelunker.cave-explorer.org/#!middleauth+https://global.daf-apis.com/nglstate/api/v1/5939082989404160"),
+            'character')
 })
 
 test_that("flywire url handling", {

man/find_merged_segments.Rd

@@ -30,6 +30,10 @@ test_that("decode scene works", {
   u="https://neuromancer-seung-import.appspot.com/#!%7B%22layers%22:[%7B%22source%22:%22precomputed://gs://zetta_lee_fly_vnc_001_precomputed/fanc_v4_em%22,%22type%22:%22image%22,%22blend%22:%22default%22,%22shaderControls%22:%7B%7D,%22name%22:%22FANCv4%22%7D,%7B%22source%22:%22graphene://https://cave.fanc-fly.com/segmentation/table/mar2021_prod%22,%22type%22:%22segmentation_with_graph%22,%22colorSeed%22:1792288153,%22segmentColors%22:%7B%22648518346498254576%22:%22#1fe0f9%22%7D,%22segments%22:[%22648518346498254576%22],%22skeletonRendering%22:%7B%22mode2d%22:%22lines_and_points%22,%22mode3d%22:%22lines%22%7D,%22graphOperationMarker%22:[%7B%22annotations%22:[],%22tags%22:[]%7D,%7B%22annotations%22:[],%22tags%22:[]%7D],%22pathFinder%22:%7B%22color%22:%22#ffff00%22,%22pathObject%22:%7B%22annotationPath%22:%7B%22annotations%22:[],%22tags%22:[]%7D,%22hasPath%22:false%7D%7D,%22name%22:%22seg_Mar2021_proofreading%22%7D,%7B%22source%22:%22precomputed://gs://lee-lab_female-adult-nerve-cord/alignmentV4/synapses/postsynapses_May2021%22,%22type%22:%22image%22,%22blend%22:%22default%22,%22shader%22:%22void%20main()%20%7B%20emitRGBA(vec4(1,%200,%201,%20toNormalized(getDataValue())));%20%7D%22,%22shaderControls%22:%7B%7D,%22name%22:%22synapses_May2021%22,%22visible%22:false%7D,%7B%22type%22:%22segmentation%22,%22mesh%22:%22precomputed://gs://zetta_lee_fly_vnc_001_precomputed/vnc1_full_v3align_2/brain_regions%22,%22objectAlpha%22:0.1,%22hideSegmentZero%22:false,%22ignoreSegmentInteractions%22:true,%22segmentColors%22:%7B%221%22:%22#bfbfbf%22,%222%22:%22#d343d6%22%7D,%22segments%22:[%221%22,%222%22],%22skeletonRendering%22:%7B%22mode2d%22:%22lines_and_points%22,%22mode3d%22:%22lines%22%7D,%22name%22:%22volume%20outlines%22%7D],%22navigation%22:%7B%22pose%22:%7B%22position%22:%7B%22voxelSize%22:[4.300000190734863,4.300000190734863,45],%22voxelCoordinates%22:[48848.171875,114737.2109375,2690]%7D%7D,%22zoomFactor%22:11.839474231467724%7D,%22perspectiveZoom%22:6704.002738252677,%22showSlices%22:false,%22gpuMemoryLimit%22:4000000000,%22systemMemoryLimit%22:4000000000,%22concurrentDownloads%22:64,%22jsonStateServer%22:%22https://global.daf-apis.com/nglstate/api/v1/post%22,%22selectedLayer%22:%7B%22layer%22:%22seg_Mar2021_proofreading%22,%22visible%22:true%7D,%22layout%22:%22xy-3d%22%7D"
 
   expect_s3_class(sc <- ngl_decode_scene(u), 'ngscene')
+  expect_s3_class(ngl_decode_scene("https://tinyurl.com/rmr58jpn"), 'ngscene')
+  expect_s3_class(ngl_decode_scene("https://neuroglancer-demo.appspot.com/#!gs://flyem-user-links/short/2023-08-26.151006.json"), 'ngscene')
+  expect_s3_class(ngl_decode_scene("https://tinyurl.com/flywirehb2"), 'ngscene')
+  expect_type(ngl_decode_scene("https://tinyurl.com/flywirehb2", return.json = T), 'character')
 })
 
 test_that("we can work round toJSON array issue",{