From 68b114ccbf55118e83ab248d078dc7b3276ebb6a Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 26 Jul 2023 15:04:44 +0200 Subject: [PATCH 01/11] Update process graph as an ordered set --- src/ProcessGraph.jl | 44 ++++++++++++++++---------------------------- src/Processes.jl | 11 ++++++----- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/ProcessGraph.jl b/src/ProcessGraph.jl index f6d886f..b2eed5b 100644 --- a/src/ProcessGraph.jl +++ b/src/ProcessGraph.jl @@ -1,62 +1,50 @@ using OrderedCollections -function flatten!(g::AbstractProcessNode, root_id, nodes=Vector{ProcessNode}()) - processes = filter(((k, v),) -> v isa ProcessNode, g.arguments) +function flatten!(g::AbstractProcessNode, root_id, nodes=OrderedSet{ProcessNode}()) + arguments_nodes = filter(((k, v),) -> v isa ProcessNode, g.arguments) # post order tree traversal - for (key, child) in processes + for (key, child) in arguments_nodes flatten!(child, root_id, nodes) g.arguments[key] = ProcessNodeReference(child.id) end - append!(nodes, [v for (k, v) in processes]) + union!(nodes, OrderedSet([v for (k, v) in arguments_nodes])) if g.id == root_id - return vcat(nodes, [g]) + push!(nodes, g) + return nodes end end abstract type AbstractProcessGraph end mutable struct ProcessGraph <: AbstractProcessGraph - data::OrderedDict + data::OrderedSet{ProcessNode} end StructTypes.StructType(::Type{ProcessGraph}) = StructTypes.CustomStruct() -StructTypes.lower(g::ProcessGraph) = g.data +StructTypes.lower(g::ProcessGraph) = [Symbol(x.id) => x for x in g.data] |> OrderedDict +Base.getindex(g::ProcessGraph, i...) = g.data[i...] function Base.show(io::IO, ::MIME"text/plain", g::ProcessGraph) - println(io, "openEO ProcessGraph with steps:") - for (id, step) in enumerate(values(g.data)) + println(io, "openEO ProcessGraph with $(length(g)) steps:") + for step in g.data args = join(values(step.arguments), ", ") - println(io, " $(id):\t $(step.process_id)($(args))") + println(io, " $(step.process_id)($(args))") end end function ProcessGraph(process_call::ProcessNode) g = deepcopy(process_call) + g.result = true root_id = process_call.id processes = flatten!(g, root_id) - - res = OrderedDict() - for p in processes - id = p.id - delete!(res, id) - res[p.id] = p - end - - # set last node as result - l = last(res).second - l = ProcessNode(l.id, l.process_id, l.arguments, true) - res[l.id] = l - - return ProcessGraph(res) + return ProcessGraph(processes) end -function Base.getindex(g::ProcessGraph, i) - id = g.data.keys[i] - return Base.getindex(g.data, id) -end +Base.getindex(g::ProcessGraph, i) = Base.getindex(g.data, i) +Base.length(g::ProcessGraph) = Base.length(g.data) struct Reducer <: AbstractProcessGraph process_graph::OrderedDict diff --git a/src/Processes.jl b/src/Processes.jl index 44003f5..c6806f4 100644 --- a/src/Processes.jl +++ b/src/Processes.jl @@ -49,10 +49,10 @@ struct ProcessNodeParameter <: AbstractProcessNode from_parameter::String end -struct ProcessNode <: AbstractProcessNode - id::String - process_id::String - arguments::Dict{Symbol,Any} +mutable struct ProcessNode <: AbstractProcessNode + const id::String + const process_id::String + const arguments::Dict{Symbol,Any} result::Bool end ProcessNode(id, process_id, arguments) = ProcessNode(id, process_id, arguments, false) @@ -95,8 +95,9 @@ end function Base.show(io::IO, ::MIME"text/plain", p::ProcessNode) - println(io, "openEO ProcessNode $(p.id) with parameters:") + println(io, "openEO ProcessNode $(p.id)") pretty_print(io, p.arguments) + pretty_print(io, Dict(:result => p.result)) end function get_parameters(parameters) From 828201ff20523a51a8e0248a1e9536ca351ed9e8 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 26 Jul 2023 15:25:21 +0200 Subject: [PATCH 02/11] Update CI.yml --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 553b081..d96b2fc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,7 @@ concurrency: jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} self-hosted strategy: fail-fast: false matrix: From 930dafe1e9598b22d9c64e5233e4a5b6c7ecbd9a Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 26 Jul 2023 15:29:59 +0200 Subject: [PATCH 03/11] Update CI.yml --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d96b2fc..553b081 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,7 @@ concurrency: jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} - runs-on: ${{ matrix.os }} self-hosted + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: From 831b78f6ee4b5203eae3dbda240aa7ac3a802c53 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 26 Jul 2023 16:42:27 +0200 Subject: [PATCH 04/11] Add tutorial --- docs/Project.toml | 4 ++++ docs/make.jl | 2 +- docs/src/index.md | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 714d845..761fa3d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,7 @@ [deps] +ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" OpenEOClient = "453781fe-a3c2-4c28-a19a-0c7c3d89805f" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" +ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" diff --git a/docs/make.jl b/docs/make.jl index d11125c..1026b53 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -15,7 +15,7 @@ makedocs(; assets=String[] ), pages=[ - "Home" => "index.md", + "Home" => "index.md" ] ) diff --git a/docs/src/index.md b/docs/src/index.md index 24b358d..01e7c5a 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -6,6 +6,44 @@ CurrentModule = OpenEOClient Documentation for [OpenEOClient](https://github.com/Open-EO/openeo-julia-client). + +# Tutorial + +Processing data on a openEO server requires authentication. + + +```@example tutorial +using OpenEOClient +username = ENV["username"] +password = ENV["password"] +c = connect("earthengine.openeo.org", "v1.0", username, password) +step1 = c.load_collection( + "COPERNICUS/S2", BoundingBox(west=16.06, south=48.06, east=16.65, north=48.35), + ["2020-01-01", "2020-01-31"], ["B10"] +) +step2 = c.reduce_dimension(step1, Reducer("median"), "t", nothing) +step3 = c.save_result(step2, "GTIFF-ZIP", Dict()) +path = c.compute_result(step3) +``` + +The data is downloaded in zipped GeoTiff format. +It can be loaded into a local Julia session using [Rasters.jl](https://rafaqz.github.io/Rasters.jl/stable/): + +```@example tutorial +using ZipFile, Rasters, Plots, ArchGDAL + +f = ZipFile.Reader(path).files[1] +write(open(f.name, "w"), read(f, String)) + +cube = Raster(f.name) +``` + +Plotting: + +```@example tutorial +plot(cube) +``` + ```@index ``` From 10d4f035f18f09d9a26ac7771d6b7e31a183bf7c Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 28 Jul 2023 11:47:57 +0200 Subject: [PATCH 05/11] Update processes: Parse all schemata --- Project.toml | 1 - src/Processes.jl | 54 +++++++++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/Project.toml b/Project.toml index 897ed37..b72382d 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" -Infiltrator = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" diff --git a/src/Processes.jl b/src/Processes.jl index c6806f4..1fe9866 100644 --- a/src/Processes.jl +++ b/src/Processes.jl @@ -59,11 +59,24 @@ ProcessNode(id, process_id, arguments) = ProcessNode(id, process_id, arguments, StructTypes.StructType(::Type{ProcessNode}) = StructTypes.Mutable() StructTypes.excludes(::Type{ProcessNode}) = (:id,) -function ProcessNode(process_id::String, parameters) - id = (process_id, parameters) |> repr |> objectid |> base64encode |> x -> process_id * "_" * x +function ProcessNode(process_id::String, parameters; id_annotation="") + id_hash = (process_id, parameters) |> repr |> objectid |> base64encode + id = [process_id, id_annotation, id_hash] |> filter(!isempty) |> x -> join(x, "_") ProcessNode(id, process_id, parameters) end +# to transpile julia method calls to openEO process graphs +function ProcessNode(e::Expr, lowered::Core.CodeInfo) + arguments = Dict( + :data => Dict(:from_parameter => "data"), + :index => e.args[2].args[3] + ) + slot = e.args[1] |> Symbol |> String |> x -> x[2:end] |> x -> parse(Int64, x) + id_annotation = String(lowered.slotnames[slot]) + p = ProcessNode("array_element", arguments; id_annotation=id_annotation) + return p +end + keywords = [ # julia keywords "begin", "while", "if", "for", "try", "return", "break", "continue", @@ -102,12 +115,12 @@ end function get_parameters(parameters) # openEO type string to Julia type - openeo_types = Dict( + julia_types_map = Dict( "string" => String, "boolean" => Bool, "number" => Number, "integer" => Integer, - "object" => Any, + "object" => Dict, "null" => Nothing, "array" => Vector, # subtypes @@ -119,28 +132,21 @@ function get_parameters(parameters) res = [] # result must be ordered for p in parameters name = Symbol(p.name) - # implement first method of function - # TODO: Multiple dispatch on other methods - schema = p.schema - schema = typeof(schema) <: Vector ? schema[1] : schema - - if "subtype" in keys(schema) && schema["subtype"] in keys(openeo_types) - openeo_type = schema["subtype"] - # TODO: can be multiple return types - elseif "type" in keys(schema) && schema["type"] in keys(openeo_types) - openeo_type = schema["type"] - else - openeo_type = "object" - end - - if typeof(openeo_type) <: Vector - types = map(x -> openeo_types[x], openeo_type) - type = eval(Meta.parse("Union{" * join(types, ",") * "}")) - else - type = openeo_types[openeo_type] + schemata = p.schema isa Vector ? p.schema : [p.schema] + types = [] + for s in schemata + if "subtype" in keys(s) + push!(types, s["subtype"]) + elseif "type" in keys(s) + push!(types, s["type"]) + else + push!(types, "object") + end end + julia_types = [get(julia_types_map, t, String) for t in types] + julia_type = eval(Meta.parse("Union{" * join(julia_types, ",") * "}")) - append!(res, [(name => type)]) + push!(res, name => julia_type) end return res end From 63087c85cecaed3aebf09a0fbb7287d1e60fb7ff Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 28 Jul 2023 11:54:05 +0200 Subject: [PATCH 06/11] Fix Infiltrator --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index b72382d..897ed37 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +Infiltrator = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" From e8dd4ccf2fd4c4df59643021225aada76a733faf Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 28 Jul 2023 13:03:22 +0200 Subject: [PATCH 07/11] Fix openEO type object --- src/Processes.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processes.jl b/src/Processes.jl index 1fe9866..9c9d2e0 100644 --- a/src/Processes.jl +++ b/src/Processes.jl @@ -120,7 +120,7 @@ function get_parameters(parameters) "boolean" => Bool, "number" => Number, "integer" => Integer, - "object" => Dict, + "object" => Any, "null" => Nothing, "array" => Vector, # subtypes @@ -135,7 +135,7 @@ function get_parameters(parameters) schemata = p.schema isa Vector ? p.schema : [p.schema] types = [] for s in schemata - if "subtype" in keys(s) + if "subtype" in keys(s) && s["subtype"] in keys(julia_types_map) push!(types, s["subtype"]) elseif "type" in keys(s) push!(types, s["type"]) From 8d666b2c42408ead032cc29f12a580c9dfbb8a20 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 28 Jul 2023 15:30:26 +0200 Subject: [PATCH 08/11] Add user defined process see #19 --- src/ProcessGraph.jl | 74 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/src/ProcessGraph.jl b/src/ProcessGraph.jl index b2eed5b..5961225 100644 --- a/src/ProcessGraph.jl +++ b/src/ProcessGraph.jl @@ -47,7 +47,7 @@ Base.getindex(g::ProcessGraph, i) = Base.getindex(g.data, i) Base.length(g::ProcessGraph) = Base.length(g.data) struct Reducer <: AbstractProcessGraph - process_graph::OrderedDict + process_graph::Union{OrderedDict,AbstractProcessGraph,Dict} end """ @@ -62,6 +62,78 @@ function Reducer(process::String="mean") return Reducer(process_graph) end +function action(n::Union{Expr,Core.ReturnNode}) + typeof(n) == Core.ReturnNode && return :noop + try + # BUG: can not disinglush between array in function argument and new ones inside the function + n.args[2].args[1].name == :getindex && return :create + catch + end + n.head == :(=) && return :assign + return :call +end + +openeo_processes = Dict( + :+ => "add", + :- => "subtract", + :* => "multiply", + :/ => "divide" +) + +""" +Create a ProcessGraph using a user-defined Function + +This is useful e.g. to create reducers to combine values of different bands in a customized way. +""" +function ProcessGraph(func::Function, types::Tuple{DataType}=(Any,)) + lowered = code_lowered(func, types)[1] + actions = action.(lowered.code) + + idx = 0 # openEO arrays starts with 0 + array_processes = Dict() + for (step, action) in zip(lowered.code, actions) + if action == :create + p = ProcessNode("array_element", Dict( + :data => Dict(:from_parameter => "data"), + :index => idx + )) + idx += 1 + push!(array_processes, step.args[1].id => p) + end + end + + processes = Vector{ProcessNode}() + for (step, action) in zip(lowered.code, actions) + if action == :create + push!(processes, array_processes[step.args[1].id]) + elseif action == :call + process_id = openeo_processes[step.args[1].name] + arguments = Dict( + :x => array_processes[step.args[2].id], + :y => array_processes[step.args[3].id] + ) + process = ProcessNode(process_id, arguments) + push!(processes, process) + elseif action == :assign + step = step.args[2].args + process_id = openeo_processes[step[1].name] + arguments = Dict( + :x => processes[step[2].id], + :y => processes[step[3].id] + ) + process = ProcessNode(process_id, arguments) + push!(processes, process) + end + end + processes |> last |> ProcessGraph +end + +function Reducer(func::Function, types::Tuple{DataType}=(Any,)) + process_graph = ProcessGraph(func, types) + return Reducer(process_graph) +end + + """ Process and download data synchronously """ From ec4214d2d7927a16160fd8220592be40181018bd Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 4 Aug 2023 11:58:01 +0200 Subject: [PATCH 09/11] Add OIDC auth see #5 --- Project.toml | 2 + src/API.jl | 1 + src/Collections.jl | 8 +-- src/Connections.jl | 130 ++++++++++++++++++++++++++++++++++++-------- src/OpenEOClient.jl | 2 +- src/ProcessGraph.jl | 44 +++++---------- src/Processes.jl | 25 ++++++--- 7 files changed, 145 insertions(+), 67 deletions(-) diff --git a/Project.toml b/Project.toml index 897ed37..1f4e9ae 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,8 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Infiltrator = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" [compat] diff --git a/src/API.jl b/src/API.jl index 0fe31fc..8f3135f 100644 --- a/src/API.jl +++ b/src/API.jl @@ -33,6 +33,7 @@ end Lists all information about a specific collection specified by the identifier """ function describe_collection(connection::AbstractConnection, id::String) + # TODO: parse to collection type response = fetchApi(connection, "collections/$(id)") response isa Exception ? throw(response) : true return response diff --git a/src/Collections.jl b/src/Collections.jl index 1a4ccf5..799f77f 100644 --- a/src/Collections.jl +++ b/src/Collections.jl @@ -2,13 +2,13 @@ import StructTypes struct Collection stac_version::String - stac_extensions::Vector{Any} - type::String + stac_extensions::Any + type::Union{String,Nothing} id::String - title::String + title::Union{String,Nothing} description::String license::String - providers::Vector{Any} + providers::Any extent::Any links::Vector{Any} end diff --git a/src/Connections.jl b/src/Connections.jl index 3d3ce9c..9bc1c29 100644 --- a/src/Connections.jl +++ b/src/Connections.jl @@ -1,23 +1,92 @@ -using HTTP -using JSON3 +import SHA +import Random +import Base64 +import HTTP +import JSON3 + +"OpenID Connect device flow + PKCE" +function get_acces_token(discovery_url::String, client_id::String, scopes::AbstractVector) + oidc_config = HTTP.get(discovery_url).body |> JSON3.read + + code_verifier = Random.randstring(128) + hash = SHA.sha256(code_verifier) + # see https://datatracker.ietf.org/doc/html/rfc7636#appendix-B + code_challenge = hash |> Base64.base64encode |> x -> strip(x, '=') |> x -> replace(x, "+" => "-", "/" => "_") + + headers = ["content-type" => "application/x-www-form-urlencoded"] + args = Dict( + "client_id" => client_id, + "grant_type" => "urn:ietf:params:oauth:grant-type:device_code", + "code_challenge_method" => "S256", + "code_challenge" => code_challenge, + "scope" => join(scopes, " ") # EDIT: Use space instead of ',' + ) + body = HTTP.URIs.escapeuri(args) + device_code_request = HTTP.post(oidc_config.device_authorization_endpoint, headers, body) |> x -> x.body |> String |> JSON3.read + + @info "Please log in using any device at:\n" * + device_code_request.verification_uri_complete * + "\nWaiting until log in succeeded..." + + while true + headers = ["content-type" => "application/x-www-form-urlencoded"] + args = Dict( + "grant_type" => "urn:ietf:params:oauth:grant-type:device_code", + "device_code" => device_code_request.device_code, + "client_id" => client_id, + "code_verifier" => code_verifier + ) + body = HTTP.URIs.escapeuri(args) + + try + access_token = HTTP.post(oidc_config.token_endpoint, headers, body) |> + x -> x.body |> String |> JSON3.read |> x -> x.access_token + return access_token + catch e + sleep(device_code_request.interval) + end + end +end abstract type AbstractConnection end -abstract type AuthorizedConnection <: AbstractConnection end struct UnAuthorizedConnection <: AbstractConnection host::String version::String end -struct BasicAuthConnection <: AuthorizedConnection +Base.show(io::IO, c::UnAuthorizedConnection) = print(io, "unauthorized openEO connection to https://$(c.host)/$(c.version)") + +struct AuthorizedConnection <: AbstractConnection host::String version::String - access_token::Union{String,Nothing} + authorization::String +end + +Base.show(io::IO, c::AuthorizedConnection) = print(io, "authorized openEO connection to https://$(c.host)/$(c.version)") + +"HTTP basic authentification" +function AuthorizedConnection(host, version, username, password) + access_response = fetchApi("https://$(username):$(password)@$(host)/$(version)/credentials/basic") + access_token = access_response.access_token + access_token = get_access_token(host, version, username, password) + authorization = "Bearer basic//$access_token" + AuthorizedConnection(host, version, authorization) end -struct OpenIDConnection <: AuthorizedConnection +"OpenID Connect device flow + PKCE authentification" +function AuthorizedConnection(host, version) + provider = fetchApi("https://$(host)/$(version)/credentials/oidc").providers[1] + discovery_url = "$(provider.issuer).well-known/openid-configuration" + client_id = provider.default_clients[1].id + scopes = provider.scopes + access_token = get_acces_token(discovery_url, client_id, scopes) + authorization = "Bearer oidc/$(provider.id)/$access_token" + AuthorizedConnection(host, version, authorization) end +@enum AuthMethod no_auth basic_auth oidc_auth + const default_headers = [ "Accept" => "application/json", "Content-Type" => "application/json" @@ -31,14 +100,13 @@ function fetchApi(url; method="GET", headers=deepcopy(default_headers), output_t response_type = Dict(response.headers)["Content-Type"] if response_type == "application/json" response_string = String(response.body) - response_converted = JSON3.read(response_string, output_type) + response_converted = output_type == Any ? JSON3.read(response_string) : JSON3.read(response_string, output_type) return response_converted else return response end catch e - msg = e.response.body |> String |> JSON3.read |> x -> x.message - return ErrorException(msg) + @error e end end @@ -50,38 +118,52 @@ end function fetchApi(connection::AuthorizedConnection, path::String; headers=deepcopy(default_headers), kw...) url = "https://$(connection.host)/$(connection.version)/$(path)" - append!(headers, ["Authorization" => "Bearer basic//$(connection.access_token)"]) + append!(headers, ["Authorization" => connection.authorization]) response = fetchApi(url; headers=headers, kw...) return response end -function connect(host, version) +function connect(host, version, auth_method::AuthMethod=no_auth) processes_code = get_processes_code(host, version) - global n_existing_connections += 1 - module_str = """ - module Connection$(n_existing_connections) - using OpenEOClient - const connection = OpenEOClient.UnAuthorizedConnection("$host", "$version") - const collections = OpenEOClient.list_collections(connection) - const processes = OpenEOClient.list_processes(connection) - $processes_code + if auth_method == no_auth + module_str = """ + module Connection$(n_existing_connections) + using OpenEOClient + const connection = OpenEOClient.UnAuthorizedConnection("$host", "$version") + const collections = OpenEOClient.list_collections(connection) + const processes = OpenEOClient.list_processes(connection) + + $processes_code + end + """ + elseif auth_method == oidc_auth + module_str = """ + module Connection$(n_existing_connections) + using OpenEOClient + const connection = OpenEOClient.AuthorizedConnection("$host", "$version") + const collections = OpenEOClient.list_collections(connection) + const processes = OpenEOClient.list_processes(connection) + compute_result(p) = OpenEOClient.compute_result(connection, p) + + $processes_code + end + """ end - """ + global n_existing_connections += 1 eval(Meta.parse(module_str)) end function connect(host, version::String, username::String, password::String) access_response = fetchApi("https://$(username):$(password)@$(host)/$(version)/credentials/basic") access_token = access_response["access_token"] - processes_code = get_processes_code(host, version) - global n_existing_connections += 1 + module_str = """ module Connection$(n_existing_connections) using OpenEOClient - const connection = OpenEOClient.BasicAuthConnection("$host", "$version", "$access_token") + const connection = OpenEOClient.AuthorizedConnection("$host", "$version", "$access_token") const collections = OpenEOClient.list_collections(connection) const processes = OpenEOClient.list_processes(connection) compute_result(p) = OpenEOClient.compute_result(connection, p) @@ -89,5 +171,7 @@ function connect(host, version::String, username::String, password::String) $processes_code end """ + + global n_existing_connections += 1 eval(Meta.parse(module_str)) end \ No newline at end of file diff --git a/src/OpenEOClient.jl b/src/OpenEOClient.jl index e407bc0..b3b5432 100644 --- a/src/OpenEOClient.jl +++ b/src/OpenEOClient.jl @@ -16,7 +16,7 @@ export ProcessNode, Processes, ProcessGraph, - Reducer, + ProcessGraph, BoundingBox, compute_result end diff --git a/src/ProcessGraph.jl b/src/ProcessGraph.jl index 5961225..d74b9b4 100644 --- a/src/ProcessGraph.jl +++ b/src/ProcessGraph.jl @@ -17,49 +17,39 @@ function flatten!(g::AbstractProcessNode, root_id, nodes=OrderedSet{ProcessNode} end end -abstract type AbstractProcessGraph end - -mutable struct ProcessGraph <: AbstractProcessGraph - data::OrderedSet{ProcessNode} +mutable struct ProcessGraph + process_graph::OrderedSet{ProcessNode} end -StructTypes.StructType(::Type{ProcessGraph}) = StructTypes.CustomStruct() -StructTypes.lower(g::ProcessGraph) = [Symbol(x.id) => x for x in g.data] |> OrderedDict -Base.getindex(g::ProcessGraph, i...) = g.data[i...] +StructTypes.StructType(::Type{ProcessGraph}) = StructTypes.Struct() +# StructTypes.lower(g::ProcessGraph) = g +Base.getindex(g::ProcessGraph, i...) = g.process_graph[i...] function Base.show(io::IO, ::MIME"text/plain", g::ProcessGraph) println(io, "openEO ProcessGraph with $(length(g)) steps:") - for step in g.data + for step in g.process_graph args = join(values(step.arguments), ", ") println(io, " $(step.process_id)($(args))") end end -function ProcessGraph(process_call::ProcessNode) - g = deepcopy(process_call) +function ProcessGraph(process_node::ProcessNode) + g = deepcopy(process_node) g.result = true - root_id = process_call.id + root_id = process_node.id processes = flatten!(g, root_id) return ProcessGraph(processes) end -Base.getindex(g::ProcessGraph, i) = Base.getindex(g.data, i) -Base.length(g::ProcessGraph) = Base.length(g.data) - -struct Reducer <: AbstractProcessGraph - process_graph::Union{OrderedDict,AbstractProcessGraph,Dict} -end +Base.getindex(g::ProcessGraph, i) = Base.getindex(g.process_graph, i) +Base.length(g::ProcessGraph) = Base.length(g.process_graph) """ Create a ProcessGraph to reduce dimesnions """ -function Reducer(process::String="mean") - process_graph = Dict( - :reduce1 => ProcessNode( - "reduce1", process, Dict(:data => Dict(:from_parameter => "data")), true - ) - ) - return Reducer(process_graph) +function ProcessGraph(process::String="mean") + p = ProcessNode(process, Dict(:data => Dict(:from_parameter => "data")); result=true) + return ProcessGraph(OrderedSet([p])) end function action(n::Union{Expr,Core.ReturnNode}) @@ -128,12 +118,6 @@ function ProcessGraph(func::Function, types::Tuple{DataType}=(Any,)) processes |> last |> ProcessGraph end -function Reducer(func::Function, types::Tuple{DataType}=(Any,)) - process_graph = ProcessGraph(func, types) - return Reducer(process_graph) -end - - """ Process and download data synchronously """ diff --git a/src/Processes.jl b/src/Processes.jl index 9c9d2e0..8dabf3d 100644 --- a/src/Processes.jl +++ b/src/Processes.jl @@ -17,7 +17,7 @@ struct Process id::String summary::Union{Nothing,String} description::String - categories::Vector{String} + categories::Union{Nothing,Vector{String}} parameters::Vector{ProcessParameter} returns::Any examples::Union{Nothing,Vector{Any}} @@ -59,21 +59,21 @@ ProcessNode(id, process_id, arguments) = ProcessNode(id, process_id, arguments, StructTypes.StructType(::Type{ProcessNode}) = StructTypes.Mutable() StructTypes.excludes(::Type{ProcessNode}) = (:id,) -function ProcessNode(process_id::String, parameters; id_annotation="") +function ProcessNode(process_id::String, parameters; id_annotation::String="", result::Bool=false) id_hash = (process_id, parameters) |> repr |> objectid |> base64encode id = [process_id, id_annotation, id_hash] |> filter(!isempty) |> x -> join(x, "_") - ProcessNode(id, process_id, parameters) + ProcessNode(id, process_id, parameters, result) end # to transpile julia method calls to openEO process graphs -function ProcessNode(e::Expr, lowered::Core.CodeInfo) +function ProcessNode(e::Expr, lowered::Core.CodeInfo; result::Bool=false) arguments = Dict( :data => Dict(:from_parameter => "data"), :index => e.args[2].args[3] ) slot = e.args[1] |> Symbol |> String |> x -> x[2:end] |> x -> parse(Int64, x) id_annotation = String(lowered.slotnames[slot]) - p = ProcessNode("array_element", arguments; id_annotation=id_annotation) + p = ProcessNode("array_element", arguments; id_annotation=id_annotation, result=result) return p end @@ -126,7 +126,7 @@ function get_parameters(parameters) # subtypes "bounding-box" => BoundingBox, "raster-cube" => ProcessNode, - "process-graph" => AbstractProcessGraph + "process-graph" => ProcessGraph ) res = [] # result must be ordered @@ -165,7 +165,8 @@ function get_processes_code(host, version) " $(process.id)($(args_str))", process.description ] - doc_str = join(docs, "\n\n") + doc_str = join(docs, "\n\n") |> escape_string + code = """ \"\"\" $(doc_str) @@ -174,12 +175,18 @@ function get_processes_code(host, version) ProcessNode("$(process.id)", Dict{Symbol, Any}(($args_dict_str))) end """ - append!(processes_codes, [code]) + + if Meta.parse(code).head != :incomplete + append!(processes_codes, [code]) + else + println(code) + end + catch e append!(warnings, [(process.id => e)]) end end code = join(processes_codes, "\n") - length(warnings) > 0 && @warn join(warnings, "\n") + length(warnings) > 0 && @warn join(vcat(["Ignore processes with errors"], warnings), "\n") return code end \ No newline at end of file From 2c46cae674331024c10f2928f5e20483f15e81de Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 4 Aug 2023 12:48:53 +0200 Subject: [PATCH 10/11] Fix auth header for basic auth --- src/Connections.jl | 2 +- test/runtests.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Connections.jl b/src/Connections.jl index 9bc1c29..2f00b55 100644 --- a/src/Connections.jl +++ b/src/Connections.jl @@ -163,7 +163,7 @@ function connect(host, version::String, username::String, password::String) module_str = """ module Connection$(n_existing_connections) using OpenEOClient - const connection = OpenEOClient.AuthorizedConnection("$host", "$version", "$access_token") + const connection = OpenEOClient.AuthorizedConnection("$host", "$version", "Bearer basic//$access_token") const collections = OpenEOClient.list_collections(connection) const processes = OpenEOClient.list_processes(connection) compute_result(p) = OpenEOClient.compute_result(connection, p) diff --git a/test/runtests.jl b/test/runtests.jl index 4835fc5..648b190 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,7 +29,7 @@ password = ENV["OPENEO_PASSWORD"] @test Set(keys(step1.arguments)) == Set([:bands, :id, :spatial_extent, :temporal_extent]) @test step1.arguments[:bands] == ["B10"] - step2 = c2.reduce_dimension(step1, Reducer("median"), "t", nothing) + step2 = c2.reduce_dimension(step1, ProcessGraph("median"), "t", nothing) step3 = c2.save_result(step2, "JPEG", Dict()) result = c2.compute_result(step3) @test result == "out.jpeg" From 5e5ec24c9ed3971d224f5d329be69bbf3c3ac7a3 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Fri, 4 Aug 2023 14:31:44 +0200 Subject: [PATCH 11/11] Fix json serialization --- src/API.jl | 6 +++++- src/ProcessGraph.jl | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/API.jl b/src/API.jl index 8f3135f..7d39c77 100644 --- a/src/API.jl +++ b/src/API.jl @@ -45,4 +45,8 @@ Base.@kwdef struct BoundingBox{T<:Real} east::T north::T end -StructTypes.StructType(::Type{BoundingBox}) = StructTypes.Struct() \ No newline at end of file +StructTypes.StructType(::Type{BoundingBox}) = StructTypes.Struct() + +function print_json(x) + x |> JSON3.write |> JSON3.read |> JSON3.pretty +end \ No newline at end of file diff --git a/src/ProcessGraph.jl b/src/ProcessGraph.jl index d74b9b4..551b864 100644 --- a/src/ProcessGraph.jl +++ b/src/ProcessGraph.jl @@ -21,8 +21,14 @@ mutable struct ProcessGraph process_graph::OrderedSet{ProcessNode} end -StructTypes.StructType(::Type{ProcessGraph}) = StructTypes.Struct() -# StructTypes.lower(g::ProcessGraph) = g +StructTypes.StructType(::Type{ProcessGraph}) = StructTypes.CustomStruct() +function StructTypes.lower(g::ProcessGraph) + res = Dict() + for s in g.process_graph + res[s.id] = s + end + return Dict(:process_graph => res) +end Base.getindex(g::ProcessGraph, i...) = g.process_graph[i...] function Base.show(io::IO, ::MIME"text/plain", g::ProcessGraph) @@ -123,10 +129,7 @@ Process and download data synchronously """ function compute_result(connection::AuthorizedConnection, process_graph::ProcessGraph, filepath::String="", kw...) query = Dict( - :process => Dict( - :process_graph => process_graph, - :parameters => [] - ) + :process => process_graph ) headers = [