diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bfbfbb0..553b081 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,6 @@ jobs: - ubuntu-latest arch: - x64 - - x86 steps: - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 diff --git a/Project.toml b/Project.toml index 302414c..dea0b38 100644 --- a/Project.toml +++ b/Project.toml @@ -4,14 +4,16 @@ authors = ["Daniel Loos and contributors"] version = "1.0.0-DEV" [deps] +AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +Infiltrator = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" [compat] -DataFrames = "1" HTTP = "1" JSON3 = "1" StructTypes = "1" diff --git a/src/API.jl b/src/API.jl index 8357c93..32c405a 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1,11 +1,10 @@ using JSON3 -using DataFrames """ Lists all predefined processes and returns detailed process descriptions, including parameters and return values. """ function list_processes(connection::AbstractConnection) - response = fetchApi(connection, "processes"; output_type=OpenEOApiTypes.ProcessesRoot) + response = fetchApi(connection, "processes"; output_type=ProcessesRoot) return response.processes end @@ -22,7 +21,7 @@ end Lists available collections with at least the required information. """ function list_collections(connection::AbstractConnection) - response = fetchApi(connection, "collections"; output_type=OpenEOApiTypes.CollectionsRoot) + response = fetchApi(connection, "collections"; output_type=CollectionsRoot) collections = response.collections return collections end @@ -35,19 +34,10 @@ function describe_collection(connection::AbstractConnection, id::String) return response end -""" -Process and download data synchronously -""" -function save_result(connection::AbstractConnection, process_graph::Dict{<:Any}) - query = Dict( - "process" => Dict( - "process_graph" => process_graph - ) - ) - headers = [ - "Accept" => "*", - "Content-Type" => "application/json" - ] - response = fetchApi(connection, "result", "POST", headers, json(query)) - return response -end \ No newline at end of file +Base.@kwdef struct BoundingBox{T<:Real} + west::T + south::T + east::T + north::T +end +StructTypes.StructType(::Type{BoundingBox}) = StructTypes.Struct() \ No newline at end of file diff --git a/src/Collections.jl b/src/Collections.jl new file mode 100644 index 0000000..1a4ccf5 --- /dev/null +++ b/src/Collections.jl @@ -0,0 +1,35 @@ +import StructTypes + +struct Collection + stac_version::String + stac_extensions::Vector{Any} + type::String + id::String + title::String + description::String + license::String + providers::Vector{Any} + extent::Any + links::Vector{Any} +end +StructTypes.StructType(::Type{Collection}) = StructTypes.Struct() + +function Base.show(io::IO, ::MIME"text/plain", c::Collection) + println(io, "openEO Collection \"$(c.id)\"") + print(io, c.description) +end + +function Base.show(io::IO, ::MIME"text/plain", v::Vector{Collection}; n=10) + println(io, "$(length(v))-element $(typeof(v)):") + for c in first(v, n) + println(io, " $(c.id): $(c.title)") + end + n < length(v) && print("⋮") +end + +# @see https://earthengine.openeo.org/v1.0/collections +struct CollectionsRoot + collections::Vector{Collection} + links::Vector{Any} +end +StructTypes.StructType(::Type{CollectionsRoot}) = StructTypes.Struct() diff --git a/src/Connections.jl b/src/Connections.jl index 7948088..134cda1 100644 --- a/src/Connections.jl +++ b/src/Connections.jl @@ -25,15 +25,20 @@ const default_headers = [ n_existing_connections = 0 -function fetchApi(url; method="GET", headers=deepcopy(default_headers), output_type::Type=Any, kw...) - response = HTTP.request(method, url, headers) - response_type = Dict(response.headers)["Content-Type"] - if response_type == "application/json" - response_string = String(response.body) - response_converted = JSON3.read(response_string, output_type) - return response_converted - else - return response +function fetchApi(url; method="GET", headers=deepcopy(default_headers), output_type::Type=Any, body::String="", kw...) + try + response = HTTP.request(method, url, headers, body; kw...) + response_type = Dict(response.headers)["Content-Type"] + if response_type == "application/json" + response_string = String(response.body) + response_converted = JSON3.read(response_string, output_type) + return response_converted + else + return response + end + catch e + # make concise error message, not entire callstack + @error e end end diff --git a/src/OpenEOApiTypes.jl b/src/OpenEOApiTypes.jl deleted file mode 100644 index 1837815..0000000 --- a/src/OpenEOApiTypes.jl +++ /dev/null @@ -1,72 +0,0 @@ -# -# Types used by JSON3.jl to parse API responses. -# @see https://api.openeo.org/#tag/Process-Discovery/operation/list-processes -# Union{Nothing,T} describes optional elements -# - -module OpenEOApiTypes -import StructTypes - -const API_VERSION = "1.0.0" - -# -# Collections -# - -struct Collection - stac_version::String - stac_extensions::Vector{Any} - type::String - id::String - title::String - description::String - license::String - providers::Vector{Any} - extent::Any - links::Vector{Any} -end -StructTypes.StructType(::Type{Collection}) = StructTypes.Struct() - -# root e.g. https://earthengine.openeo.org/v1.0/collections -struct CollectionsRoot - collections::Vector{Collection} - links::Vector{Any} -end -StructTypes.StructType(::Type{CollectionsRoot}) = StructTypes.Struct() - -# -# Processes -# - -struct Parameter - name::String - description::String - schema::Any - default::Union{Nothing,Any} - optional::Union{Nothing,Bool} - deprecated::Union{Nothing,Bool} - experimental::Union{Nothing,Bool} -end -StructTypes.StructType(::Type{Parameter}) = StructTypes.Struct() - -struct Process - id::String - summary::String - description::String - categories::Vector{String} - parameters::Vector{Parameter} - returns::Any - examples::Union{Nothing,Vector{Any}} - links::Union{Nothing,Vector{Any}} - exceptions::Any - experimental::Union{Nothing,Bool} -end -StructTypes.StructType(::Type{Process}) = StructTypes.Struct() - -# root e.g. https://earthengine.openeo.org/v1.0/processes -struct ProcessesRoot - processes::Vector{Process} - links::Vector{Any} -end -StructTypes.StructType(::Type{ProcessesRoot}) = StructTypes.Struct() -end \ No newline at end of file diff --git a/src/OpenEOClient.jl b/src/OpenEOClient.jl index 9d459b6..d17f2cb 100644 --- a/src/OpenEOClient.jl +++ b/src/OpenEOClient.jl @@ -1,8 +1,9 @@ module OpenEOClient -include("OpenEOApiTypes.jl") include("Connections.jl") include("Processes.jl") +include("ProcessGraph.jl") +include("Collections.jl") include("API.jl") export @@ -12,7 +13,8 @@ export list_jobs, list_processes, register_processes, - ProcessCall, + ProcessNode, Processes, - save_result + BoundingBox, + compute_result end diff --git a/src/ProcessGraph.jl b/src/ProcessGraph.jl new file mode 100644 index 0000000..4b396ac --- /dev/null +++ b/src/ProcessGraph.jl @@ -0,0 +1,68 @@ +using OrderedCollections + +using Infiltrator + +function flatten!(g::AbstractProcessNode, root_id, nodes=Vector{ProcessNode}()) + processes = filter(((k, v),) -> v isa ProcessNode, g.arguments) + + # post order tree traversal + for (key, child) in processes + flatten!(child, root_id, nodes) + g.arguments[key] = ProcessNodeReference(child.id) + end + + append!(nodes, [v for (k, v) in processes]) + + if g.id == root_id + return vcat(nodes, [g]) + end +end + +function get_process_graph(process_call::ProcessNode) + g = deepcopy(process_call) + root_id = process_call.id + processes = flatten!(g, root_id) + + res = OrderedDict() + for p in processes + id = p.id + delete!(res, id) + res[p.id] = p + end + + l = last(res).second + l = ProcessNode(l.id, l.process_id, l.arguments, true) + res[l.id] = l + + return res +end + + +""" +Process and download data synchronously +""" +function compute_result(connection::AbstractConnection, process_call::ProcessNode, filepath::String="", kw...) + query = Dict( + :process => Dict( + :process_graph => get_process_graph(process_call), + :parameters => [] + ) + ) + + headers = [ + "Accept" => "*", + "Content-Type" => "application/json" + ] + + response = fetchApi(connection, "result"; method="POST", headers=headers, body=JSON3.write(query)) + + if isempty(filepath) + file_extension = split(Dict(response.headers)["Content-Type"], "/")[2] + filepath = "out." * file_extension + end + + write(open(filepath, "w"), response.body) + return filepath +end + + diff --git a/src/Processes.jl b/src/Processes.jl index ad33c65..c68556b 100644 --- a/src/Processes.jl +++ b/src/Processes.jl @@ -1,8 +1,61 @@ using HTTP +import StructTypes +import Base64: base64encode -struct ProcessCall +struct ProcessParameter + name::String + description::String + schema::Any + default::Union{Nothing,Any} + optional::Union{Nothing,Bool} + deprecated::Union{Nothing,Bool} + experimental::Union{Nothing,Bool} +end +StructTypes.StructType(::Type{ProcessParameter}) = StructTypes.Struct() + +struct Process + id::String + summary::Union{Nothing,String} + description::String + categories::Vector{String} + parameters::Vector{ProcessParameter} + returns::Any + examples::Union{Nothing,Vector{Any}} + links::Union{Nothing,Vector{Any}} + exceptions::Any + experimental::Union{Nothing,Bool} +end +StructTypes.StructType(::Type{Process}) = StructTypes.Struct() + +function Base.show(io::IO, ::MIME"text/plain", p::Process) + print(io, "$(p.id)($(join([x.name for x in p.parameters], ", "))): $(p.summary)") +end + +# root e.g. https://earthengine.openeo.org/v1.0/processes +struct ProcessesRoot + processes::Vector{Process} + links::Vector{Any} +end + +StructTypes.StructType(::Type{ProcessesRoot}) = StructTypes.Struct() + +abstract type AbstractProcessNode end + +struct ProcessNodeReference <: AbstractProcessNode + from_node::String +end + +struct ProcessNode <: AbstractProcessNode id::String - parameters + process_id::String + arguments::Dict{Symbol,Any} + result::Bool +end +ProcessNode(id, process_id, arguments) = ProcessNode(id, process_id, arguments, false) + +function ProcessNode(process_id::String, parameters) + id = (process_id, parameters) |> repr |> objectid |> base64encode |> x -> process_id * "_" * x + ProcessNode(id, process_id, parameters) end keywords = [ @@ -17,22 +70,23 @@ function pretty_print(io, d::AbstractDict, tabwidth=3) return end - max_pad = maximum([length(x) for x in keys(d)]) + 1 + max_pad = maximum([length(String(x)) for x in keys(d)]) + 1 for (k, v) in d if typeof(v) <: AbstractDict s = "$(k): " println(io, join(fill(" ", tabwidth)) * s) pretty_print(io, v, tabwidth + tabwidth) else - println(io, join(fill(" ", tabwidth)) * "$(rpad(k*":", max_pad)) $(repr(v))") + println(io, join(fill(" ", tabwidth)) * "$(rpad(String(k)*":", max_pad)) $(repr(v))") end end return nothing end -function Base.show(io::IO, ::MIME"text/plain", p::ProcessCall) + +function Base.show(io::IO, ::MIME"text/plain", p::ProcessNode) println(io, "openEO ProcessCall $(p.id) with parameters:") - pretty_print(io, p.parameters) + pretty_print(io, p.arguments) end function get_parameters(parameters) @@ -46,7 +100,8 @@ function get_parameters(parameters) "null" => Nothing, "array" => Vector, # subtypes - "bounding-box" => NTuple{4,<:Number} + "bounding-box" => BoundingBox, + "raster-cube" => ProcessNode ) res = [] # result must be ordered @@ -78,26 +133,6 @@ function get_parameters(parameters) return res end -function get_process_function(process_specs) - parameters = get_parameters(process_specs.parameters) - args_str = join(["$(k)::$(v)" for (k, v) in parameters], ", ") - args_dict_str = join([":$k=>$k" for (k, v) in parameters], ", ") - docs = [ - " $(process_specs.id)($(args_str)) -> Process", - process_specs.description - ] - doc_str = join(docs, "\n\n") - code = """ - \"\"\" - $(doc_str) - \"\"\" - function $(process_specs.id)$(process_specs.id in keywords ? "_" : "")($args_str) - ProcessCall("$(process_specs.id)", Dict(($args_dict_str))) - end - """ - return code -end - function get_processes_code(host, version) connection = UnAuthorizedConnection(host, version) processes = list_processes(connection) @@ -105,12 +140,28 @@ function get_processes_code(host, version) warnings = [] for process in processes try - code = get_process_function(process) + arguments = get_parameters(process.parameters) + args_str = join(["$(k)::$(v)" for (k, v) in arguments], ", ") + args_dict_str = join([":$k=>$k" for (k, v) in arguments], ", ") + docs = [ + " $(process.id)($(args_str))", + process.description + ] + doc_str = join(docs, "\n\n") + code = """ + \"\"\" + $(doc_str) + \"\"\" + function $(process.id)$(process.id in keywords ? "_" : "")($args_str) + ProcessNode("$(process.id)", Dict{Symbol, Any}(($args_dict_str))) + end + """ append!(processes_codes, [code]) catch e append!(warnings, [(process.id => e)]) end end code = join(processes_codes, "\n") + length(warnings) > 0 && @warn join(warnings, "\n") return code end \ No newline at end of file diff --git a/test/example1-nested.json b/test/example1-nested.json new file mode 100644 index 0000000..b77b1ab --- /dev/null +++ b/test/example1-nested.json @@ -0,0 +1,40 @@ +{ + "loadcollection1": { + "process_id": "load_collection", + "arguments": { + "bands": ["B10"], + "id": "COPERNICUS/S2", + "spatial_extent": { + "west": 16.06, + "south": 48.06, + "east": 16.65, + "north": 48.35 + }, + "temporal_extent": ["2020-01-20", "2020-01-30"] + } + }, + "saveresult1": { + "process_id": "save_result", + "arguments": { + "data": { + "process_id": "reduce_dimension", + "arguments": { + "data": { "from_node": "loadcollection1" }, + "dimension": "t", + "reducer": { + "process_graph": { + "min1": { + "process_id": "min", + "arguments": { "data": { "from_parameter": "data" } }, + "result": true + } + } + } + } + }, + "format": "JPEG", + "options": {} + }, + "result": true + } +} diff --git a/test/sentinel2-time-min.json b/test/example1-ref.json similarity index 100% rename from test/sentinel2-time-min.json rename to test/example1-ref.json diff --git a/test/runtests.jl b/test/runtests.jl index 6be7d4e..bfd41a8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,16 +12,24 @@ password = ENV["OPENEO_PASSWORD"] @test "api_version" in keys(response) @test "backend_version" in keys(response) + list_processes(unauth_con) + list_collections(unauth_con) + c1 = connect(host, version) c2 = connect(host, version, username, password) @test allequal([c1, c2] .|> x -> size(x.collections)) @test allequal([c1, c2] .|> x -> names(x, all=true) |> length) - step1 = c1.load_collection( - "COPERNICUS/S2", (16.06, 48.06, 16.65, 48.35), + step1 = c2.load_collection( + "COPERNICUS/S2", BoundingBox(west=16.06, south=48.06, east=16.65, north=48.35), ["2020-01-20", "2020-01-30"], ["B10"] ) - @test step1.id == "load_collection" - @test Set(keys(step1.parameters)) == Set([:bands, :id, :spatial_extent, :temporal_extent]) - @test step1.parameters[:bands] == ["B10"] + @test step1.id == "load_collection_tQ79zrFEGi8=" + @test step1.process_id == "load_collection" + @test Set(keys(step1.arguments)) == Set([:bands, :id, :spatial_extent, :temporal_extent]) + @test step1.arguments[:bands] == ["B10"] + + step2 = c2.save_result(step1, "GTIFF-ZIP", Dict()) + result = compute_result(c2.connection, step2) + @test result == "out.zip" end