diff --git a/DESCRIPTION b/DESCRIPTION index 742a83e..30bcb74 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,12 @@ Package: openEO.R.Backend Title: Reference backend implementation for openEO conformant backend with a local filesystem -Version: 0.2.4 -Authors@R: person("Florian", "Lahn", email = "florian.lahn@uni-muenster.de", role = c("aut", "cre")) +Version: 0.2.5 +Authors@R: c(person("Florian", "Lahn", email = "florian.lahn@uni-muenster.de", role = c("aut", "cre")), + person("Pramit","Ghosh",email = "pramitghosh@uni-muenster.de", role = c("aut","ctb"))) Description: The package contains a backend solution in compliance with the openEO API. In this demonstration the file backend solution is the local file system containing some raster data collections. +URL: https://github.com/Open-EO/openeo-r-backend +BugReports: https://github.com/Open-EO/openeo-r-backend/issues Depends: R (>= 3.3), rgdal (>= 1.2-16), @@ -27,7 +30,7 @@ Encoding: UTF-8 LazyData: true RoxygenNote: 6.0.1 VignetteBuilder: knitr -License: Apache License +License: Apache License 2.0 Imports: plumber, jsonlite, diff --git a/DOCKERFILE b/DOCKERFILE index 5527520..9dd9dac 100644 --- a/DOCKERFILE +++ b/DOCKERFILE @@ -1,11 +1,10 @@ -FROM openeo-baseserver:1.4 +FROM openeo-baseserver:1.5 MAINTAINER Florian Lahn (florian.lahn@uni-muenster.de) -LABEL version="0.2.4" +LABEL version="0.2.5" LABEL description="A simple openeo (almost) conformant backend for frontend development" # create the path for the user files -RUN mkdir -p /opt/dockerfiles/ -RUN mkdir -p /var/openeo/workspace/ +RUN mkdir -p /opt/dockerfiles/ && mkdir -p /var/openeo/workspace/ COPY ./ /opt/dockerfiles/ diff --git a/DOCKERFILE-base b/DOCKERFILE-base index 75357fa..297ae77 100644 --- a/DOCKERFILE-base +++ b/DOCKERFILE-base @@ -1,11 +1,10 @@ -FROM r-base:3.4.4 +FROM r-base:3.5.0 MAINTAINER Florian Lahn (florian.lahn@uni-muenster.de) -LABEL version="1.4" +LABEL version="1.5" LABEL description="The basic configuration of the openeo r server image" # create the path for the user files -RUN mkdir -p /opt/dockerfiles/ -RUN mkdir -p /var/openeo/workspace/ +RUN mkdir -p /opt/dockerfiles/ && mkdir -p /var/openeo/workspace/ COPY ./ /opt/dockerfiles/ diff --git a/Dockerfiles/install_package.R b/Dockerfiles/install_package.R index 9ad06f8..cbf0059 100644 --- a/Dockerfiles/install_package.R +++ b/Dockerfiles/install_package.R @@ -1,3 +1,7 @@ library(devtools) +# install the R UDF package from github +install_github("pramitghosh/OpenEO.R.UDF",ref="v0.0.1",dependencies=TRUE) + +# install the R back-end package install("/opt/dockerfiles") diff --git a/Dockerfiles/install_package_dependencies.R b/Dockerfiles/install_package_dependencies.R index d44510f..460e03d 100644 --- a/Dockerfiles/install_package_dependencies.R +++ b/Dockerfiles/install_package_dependencies.R @@ -5,4 +5,8 @@ library(devtools) install_deps("/opt/dockerfiles",repos=cran.mirror) installation = list.files("/opt/dockerfiles",recursive=TRUE,full.names = TRUE) + +# packages for the R UDFs +install_github("r-spatial/stars", dependencies=TRUE) + removed = file.remove(installation) diff --git a/NAMESPACE b/NAMESPACE index 0a80444..98653fd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -38,9 +38,13 @@ export(is.Process) export(is.Product) export(is.Service) export(is.User) +export(loadSentinel2Data) +export(raster_collection_export) export(read_legend) export(serializer_proxy) +export(udf_export) export(write_generics) +exportMethods(crs) exportMethods(extent) import(DBI) import(plumber) diff --git a/R/Collection-class.R b/R/Collection-class.R index 98a6090..1633bd5 100644 --- a/R/Collection-class.R +++ b/R/Collection-class.R @@ -105,7 +105,6 @@ Collection <- R6Class( }, addGranule = function(space=NULL,time=NULL, band=NULL, data, ..., meta_band = NULL) { - dot_args = list(...) if (is.character(data)) { @@ -188,15 +187,19 @@ Collection <- R6Class( } - if (!is.null(band) && length(band) > 1) { + if (!is.null(band) && length(band) >= 1) { #unstack also one banded images # add multiple bands - bands = unstack(data) adding = tibble(band = band) if (self$dimensions$time) { adding = add_column(adding,time = time) } - layer = unstack(data) + if (class(data) != "RasterLayer") { + layer = unstack(data) + } else { + layer = list(data) + } + adding = add_column(adding,data=layer) if (self$dimensions$space) { @@ -326,9 +329,6 @@ Collection <- R6Class( if (self$dimensions$time) { # order by bands if present, stack single bands into one file - # space is fixed (for now) - # TODO rework the holding of spatial extents in a spatial* data.frame where we store the feature and - # state the IDs in the table if (self$dimensions$band) { private$data_table = private$data_table %>% group_by(time,space) %>% @@ -433,7 +433,6 @@ Collection <- R6Class( } if (self$dimensions$time) { - # TODO group also by band! currently assuming that we have only one attribute--hmm maybe it doesn't matter out = private$data_table %>% dplyr::group_by(space) %>% dplyr::arrange(time) %>% dplyr::summarise(data=tibble(band,time,data) %>% (function(x, ...){ values = unlist(x$data) names = paste(x$band,as.character(x$time), sep=".") @@ -626,12 +625,10 @@ read_legend = function(legend.path,code, ...) { # prepareBands parentFolder = dirname(legend.path) - bandinfo = table %>% group_by(band_index) %>% summarise(band = first(band), data = first(filename)) %>% arrange(band_index) + bandinfo = table %>% group_by(band_index) %>% dplyr::summarise(band = first(band), data = first(filename)) %>% dplyr::arrange(band_index) # use band as band_index dots$band_id = bandinfo$band - - for (i in 1:nrow(table)) { row = table[i,] if (as.integer(row$whether_raster) == 1) { @@ -645,3 +642,18 @@ read_legend = function(legend.path,code, ...) { return(collection) } + +#' @export +setMethod("crs", signature="Collection", function(x, ...) { + return(crs.Collection(x,...)) +}) + +crs.Collection = function(x, ...) { + if (x$dimensions$space) { + return(x$getGlobalSRS()) + } else { + stop("Try to select a spatial reference system without a spatial dimension defined.") + } + +} + diff --git a/R/Server-class.R b/R/Server-class.R index a552cfe..83eaa55 100644 --- a/R/Server-class.R +++ b/R/Server-class.R @@ -31,8 +31,11 @@ OpenEOServer <- R6Class( workspaces.path = NULL, sqlite.path = NULL, + udf_transactions.path = NULL, + api.port = NULL, host = NULL, + baseserver.url = "http:localhost:8000/api/", mapserver.url = NULL, #assuming here a url, if not specified the backend is probably started with docker-compose processes = NULL, @@ -195,13 +198,21 @@ OpenEOServer <- R6Class( if (is.null(self$workspaces.path)) { self$workspaces.path <- getwd() } + if (is.null(self$data.path)) { self$data.path <- paste(self$workspaces.path,"data",sep="/") - - if (!dir.exists(self$data.path)) { - dir.create(self$data.path,recursive = TRUE) - } } + if (!dir.exists(self$data.path)) { + dir.create(self$data.path,recursive = TRUE) + } + + if (is.null(self$udf_transactions.path)) { + self$udf_transactions.path = paste(self$workspaces.path,"udf",sep="/") + } + if (!dir.exists(self$udf_transactions.path)) { + dir.create(self$udf_transactions.path, recursive = TRUE) + } + if (is.null(self$secret.key)) { self$secret.key <- sha256(charToRaw("openEO-R")) } diff --git a/R/api.R b/R/api.R index 3fb6af7..5c2203c 100644 --- a/R/api.R +++ b/R/api.R @@ -41,6 +41,7 @@ "/data/", "/data/{product_id}", "/jobs/", + "/jobs/{job_id}", "/jobs/{job_id}/download", "/jobs/{job_id}/queue", "/processes/", @@ -112,7 +113,7 @@ } }, error=function(e) { - error(res,403,"Login failed.") + openEO.R.Backend:::error(res,403,"Login failed.") } ) } @@ -144,16 +145,19 @@ } - job = Job$new(process_graph=process_graph,user_id = req$user$user_id) - - job = job$run() - - result = job$result - if (is.null(result)) { - openEO.R.Backend:::error(res,status = 500,msg = "The result was NULL due to an internal error during processing.") - } - - return(.create_output(res = res,result = job$results, format = format)) + tryCatch({ + job = Job$new(process_graph=process_graph,user_id = req$user$user_id) + + job = job$run() + + if (is.null(job$results)) { + return(openEO.R.Backend:::error(res,status = 500,msg = "The result was NULL due to an internal error during processing.")) + } + + return(.create_output(res = res,result = job$results, format = format)) + }, error= function(e) { + return(openEO.R.Backend:::error(res=res, status = 500, msg = e)) + }) } .ogrExtension = function(format) { @@ -236,7 +240,7 @@ } }, error = function(e) { - error(res,401,"Unauthorized") + openEO.R.Backend:::error(res,401,"Unauthorized") } ) } diff --git a/R/api_job.R b/R/api_job.R index 1fbec29..0ad070f 100644 --- a/R/api_job.R +++ b/R/api_job.R @@ -155,7 +155,7 @@ createJobsEndpoint = function() { } else { job_results = paste(openeo.server$workspaces.path,"jobs",job_id,sep="/") - base_url = paste("http://",openeo.server$host,":",openeo.server$api.port,"/api/result/",job_id,sep="") + base_url = paste(openeo.server$baseserver.url,"result/",job_id,sep="") #get files in outputfolder but not the log file paste(base_url,list.files(job_results,pattern="[^process\\.log]"),sep="/") diff --git a/R/prepare_UDF.R b/R/prepare_UDF.R index 573f2ac..9cfccb9 100644 --- a/R/prepare_UDF.R +++ b/R/prepare_UDF.R @@ -18,4 +18,142 @@ write_generics = function(collection_obj, dir_name = "disk") #dir_name could be R2G_obj$legend_to_disk(dir_name) +} + +#' Prepares the collection data for the UDF service request +#' +#' Transforms the data contained in a Collection into a JSON representation. It will be passed along the code script URL as data +#' to the specified UDF REST processing service. Currently implemented only for raster timeserires collections. +#' +#' @param collection Collection object +#' @param strategy the tiling strategy (not implemented yet) +#' @return list that can be transformed into "UdfData" JSON +#' @export +udf_export = function(collection,strategy) { + if (! is.Collection(collection)) { + stop("Passed object is not a Collection object") + } + + # TODO prepare some sort of tiling strategy + + if (collection$dimensions$raster && collection$dimensions$space && collection$dimensions$time) { + udf_data = list() + udf_data[["proj"]] = as.character(collection$getGlobalSRS()) + udf_data[["raster_collection_tiles"]] = list() + + udf_data[["raster_collection_tiles"]] = append(udf_data[["raster_collection_tiles"]],raster_collection_export(collection)) + return(udf_data) + } else { + stop("Not yet implemented") + } + +} + +udf_request = function(collection,strategy=NULL,udf_transaction) { + # TODO remove the hard coded backend selection + request = list( + code = list( + language = "R", + source = readChar(udf_transaction$script, file.info(udf_transaction$script)$size) + ), + data = udf_export(collection = collection, strategy = strategy) + ) + + return(request) +} + +#' Creates RasterCollectionTile representation +#' +#' Subsets and groups Collection data by band and space in order to create the specified UDF RasterCollectionTile JSON output. +#' +#' @param collection Collection object +#' @return list that can be transformed into "UdfData" JSON +#' @export +raster_collection_export = function(collection) { + if (! is.Collection(collection)) { + stop("Passed object is not a Collection object") + } + + data = collection$getData() + extents = collection$space + + modified = data %>% group_by(band,space) %>% dplyr::summarise( + exported = tibble(band,space,data,time) %>% (function(x,...) { + raster_collection_tiles = list() + raster_collection_tiles[["id"]] = "test1" + + raster_collection_tiles[["wavelength"]] = unique(x[["band"]]) + + # select sf polygons by ids stored in the data table, then give bbox from all of the sf + b = st_bbox(extents[x %>% dplyr::select(space) %>% unique() %>% unlist() %>% unname(),]) + raster_collection_tiles[["extent"]] = list( + north = b[["ymax"]], + south = b[["ymin"]], + west = b[["xmin"]], + east = b[["xmax"]], + height = yres(x[[1,"data"]]), + width = xres(x[[1,"data"]]) + ) + + # times + times = x[["time"]] + tres = round(median(diff(times))) + raster_collection_tiles[["start_times"]] = strftime(times, "%Y-%m-%dT%H:%M:%S", usetz = TRUE) + raster_collection_tiles[["end_times"]] = strftime(times+tres, "%Y-%m-%dT%H:%M:%S", usetz = TRUE) + + # fetch data from raster files as matrix (store as list first other wise it messes up the matrix + # structure by creating row/col as rows for each attribute) + raster_collection_tiles[["data"]] = x %>% apply(MARGIN=1,FUN= function(row) { + + return(list(raster::values(x=row$data, format="matrix"))) + + }) + + # unlist it again + raster_collection_tiles[["data"]] = lapply(raster_collection_tiles[["data"]], function(arr_list) { + arr_list[[1]] + }) + + return(list(raster_collection_tiles)) + }) + ) + return(modified[["exported"]]) +} + +prepare_udf_transaction = function(user,script) { + # TODO mayb script is URL + isURL = FALSE + + # /udf// + transaction_id = createAlphaNumericId(n=1,length=18) + + if (isURL) { + # download the script and store it in the user workspace + script.url = script + } else { + # then we need to make the script accessable as URL + file.path = paste(user$workspace,"files", script, sep="/") + } + + + udf_transaction_folder = paste(openeo.server$udf_transactions.path,transaction_id,sep="/") + + if (!dir.exists(udf_transaction_folder)) { + dir.create(udf_transaction_folder,recursive = TRUE) + } + + results.file.path = paste(udf_transaction_folder, "results", sep = "/") + if (!dir.exists(results.file.path)) { + dir.create(results.file.path,recursive = TRUE) + } + + udf_transaction = list( + id = transaction_id, + script = file.path, + input = udf_transaction_folder, + result = results.file.path + ) + class(udf_transaction) = "udf_transaction" + + return(udf_transaction) } \ No newline at end of file diff --git a/R/processes.R b/R/processes.R index 80db644..4c64099 100644 --- a/R/processes.R +++ b/R/processes.R @@ -315,52 +315,47 @@ aggregate_time = Process$new( script = gsub("^/", "", script) } + # fla: if the file is hosted at this backend # else we need to download it first. - file.path = paste(user$workspace,"files", script, sep="/") - - udf_transaction_folder = paste(openeo.server$workspaces.path,"udf",createAlphaNumericId(n=1,length=18),sep="/") - if (!dir.exists(udf_transaction_folder)) { - dir.create(udf_transaction_folder,recursive = TRUE) - } + # prepare paths + udf_transaction = prepare_udf_transaction(user,script) - results.file.path = paste(udf_transaction_folder, "results", sep = "/") - if (!dir.exists(results.file.path)) { - dir.create(results.file.path,recursive = TRUE) - } - write_generics(collection,dir_name = udf_transaction_folder) + # export data + write_generics(collection,dir_name = udf_transaction$input) + #testing + write(toJSON(udf_request(collection=collection,udf_transaction = udf_transaction),auto_unbox=TRUE,pretty = TRUE),paste(udf_transaction$input,"udf_request.json",sep="/")) oldwd = getwd() tryCatch({ - setwd(udf_transaction_folder) # TODO revise this, this is and can be only temporary! this can only work - # as long we run the code in this server application. if we create another process, this might fail - - source(file = file.path, local = TRUE) - # we need to specify where to store the results here - # fla: for run_UDF it should not be possible for an user to change the out_dir... we are currently - # blind at this point. There is nothing fix where the backend can find the results! - - + setwd(udf_transaction$input) + source(file = udf_transaction$script, local = TRUE) # Now read back results present at results.file.path # To be implemented once classes for data I/O have been re-written # The argument "code" will eventually be evaulated from the dimensions of "collection" and "modifier" # -> modification is applied afterwards # TODO replace code with something that is read from a global meta data file - result.collection = read_legend(legend.path = paste(results.file.path, "out_legend.csv", sep = "/"), code = "11110") + result.collection = read_legend(legend.path = paste(udf_transaction$result, "out_legend.csv", sep = "/"), code = "11110") return(result.collection) }, error = function(e) { cat(paste("ERROR:",e)) - },finally= function(){ - setwd(oldwd) + },finally = { # cleanup at this point the results should been written to disk already, clear export! - # unlink(udf_export_folder,recursive = TRUE) + files = list.files(path=".", recursive = TRUE,full.names = TRUE) + unlink(files[!grepl("result",files)],recursive = TRUE) + + dirs=list.dirs(".") + unlink(dirs[!grepl("result",dirs)][-1], recursive = TRUE) # -1 removes the first argument (the transaction folder) + + setwd(oldwd) + }) } @@ -396,50 +391,37 @@ apply_pixel = Process$new( # fla: if the file is hosted at this backend # else we need to download it first. - file.path = paste(user$workspace,"files", script, sep="/") - - udf_transaction_folder = paste(openeo.server$workspaces.path,"udf",createAlphaNumericId(n=1,length=18),sep="/") - - if (!dir.exists(udf_transaction_folder)) { - dir.create(udf_transaction_folder,recursive = TRUE) - } + # prepare paths + udf_transaction = prepare_udf_transaction(user,script) - results.file.path = paste(udf_transaction_folder, "results", sep = "/") - if (!dir.exists(results.file.path)) { - dir.create(results.file.path,recursive = TRUE) - } + # file.path = paste(user$workspace,"files", script, sep="/") - write_generics(collection,dir_name = udf_transaction_folder) + # export data + write_generics(collection,dir_name = udf_transaction$input) oldwd = getwd() tryCatch({ - setwd(udf_transaction_folder) # TODO revise this, this is and can be only temporary! this can only work - # as long we run the code in this server application. if we create another process, this might fail - - source(file = file.path, local = TRUE) - # we need to specify where to store the results here - # fla: for run_UDF it should not be possible for an user to change the out_dir... we are currently - # blind at this point. There is nothing fix where the backend can find the results! - - - - # Now read back results present at results.file.path - # To be implemented once classes for data I/O have been re-written - # The argument "code" will eventually be evaulated from the dimensions of "collection" and "modifier" - # -> modification is applied afterwards + setwd(udf_transaction$input) + source(file = udf_transaction$script, local = TRUE) + # TODO replace code with something that is read from a global meta data file - result.collection = read_legend(legend.path = paste(results.file.path, "out_legend.csv", sep = "/"), code = "11110") + result.collection = read_legend(legend.path = paste(udf_transaction$result, "out_legend.csv", sep = "/"), code = "11110") return(result.collection) }, error = function(e) { cat(paste("ERROR:",e)) },finally= function(){ - setwd(oldwd) # cleanup at this point the results should been written to disk already, clear export! - # unlink(udf_export_folder,recursive = TRUE) + files = list.files(path=".", recursive = TRUE,full.names = TRUE) + unlink(files[!grepl("result",files)],recursive = TRUE) + + dirs=list.dirs(".") + unlink(dirs[!grepl("result",dirs)][-1], recursive = TRUE) # -1 removes the first argument (the transaction folder) + + setwd(oldwd) }) } diff --git a/docker-compose.yml b/docker-compose.yml index a4247af..05e5f69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,12 +4,12 @@ services: build: context: . dockerfile: DOCKERFILE-base - image: openeo-baseserver:1.4 + image: openeo-baseserver:1.5 openeo-r-server: build: context: . dockerfile: DOCKERFILE - image: openeo-r-backend:0.2.4 + image: openeo-r-backend:0.2.5 container_name: openeo-r-server ports: - "8000:8000" diff --git a/man/raster_collection_export.Rd b/man/raster_collection_export.Rd new file mode 100644 index 0000000..727321e --- /dev/null +++ b/man/raster_collection_export.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prepare_UDF.R +\name{raster_collection_export} +\alias{raster_collection_export} +\title{Creates RasterCollectionTile representation} +\usage{ +raster_collection_export(collection) +} +\arguments{ +\item{collection}{Collection object} +} +\value{ +list that can be transformed into "UdfData" JSON +} +\description{ +Subsets and groups Collection data by band and space in order to create the specified UDF RasterCollectionTile JSON output. +} diff --git a/man/read_dimensionality.Rd b/man/read_dimensionality.Rd index 289ae76..50dfb5a 100644 --- a/man/read_dimensionality.Rd +++ b/man/read_dimensionality.Rd @@ -7,7 +7,7 @@ read_dimensionality(code) } \arguments{ -\item{code}{code string or integer value that represents a value in 0 to 2^5-1 at most} +\item{code}{code string or a single integer value that represents a value in 0 to 2^5-1 at most} } \value{ Dimensionality object diff --git a/man/udf_export.Rd b/man/udf_export.Rd new file mode 100644 index 0000000..6da66b9 --- /dev/null +++ b/man/udf_export.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prepare_UDF.R +\name{udf_export} +\alias{udf_export} +\title{Prepares the collection data for the UDF service request} +\usage{ +udf_export(collection, strategy) +} +\arguments{ +\item{collection}{Collection object} + +\item{strategy}{the tiling strategy (not implemented yet)} +} +\value{ +list that can be transformed into "UdfData" JSON +} +\description{ +Transforms the data contained in a Collection into a JSON representation. It will be passed along the code script URL as data +to the specified UDF REST processing service. Currently implemented only for raster timeserires collections. +}