From 21c4d62aeefe464e3d7257604f75f5e7abeff3dd Mon Sep 17 00:00:00 2001 From: Islam El-Ashi Date: Thu, 27 Jan 2022 09:07:51 +0100 Subject: [PATCH 1/9] Draft --- spec/ic.did | 23 +++++++++++++++++++++++ spec/index.adoc | 7 +++++++ 2 files changed, 30 insertions(+) diff --git a/spec/ic.did b/spec/ic.did index 5b64725d1..e608cfb7d 100644 --- a/spec/ic.did +++ b/spec/ic.did @@ -16,6 +16,20 @@ type definite_canister_settings = record { freezing_threshold : nat; }; +type http_header = record { 0: text; 1: text }; + +type http_response = record { + status: nat; + headers: vec http_header; + body: blob; +}; + +type http_request_error = variant { + no_consensus; + timeout; + bad_tls; +}; + service ic : { create_canister : (record { settings : opt canister_settings @@ -43,6 +57,15 @@ service ic : { delete_canister : (record {canister_id : canister_id}) -> (); deposit_cycles : (record {canister_id : canister_id}) -> (); raw_rand : () -> (blob); + http_request : (record { + url : text; + method : variant { get }; + headers: vec http_header; + body : opt blob; + transform : opt variant { + function: func (http_response) -> (http_response) query + }; + }) -> (variant { Ok : http_response; Err: opt http_request_error }); // provisional interfaces for the pre-ledger world provisional_create_canister_with_cycles : (record { diff --git a/spec/index.adoc b/spec/index.adoc index b081cd498..d9d56b69b 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1654,6 +1654,13 @@ There is no restriction on who can invoke this method. This method takes no input and returns 32 pseudo-random bytes to the caller. The return value is unknown to any part of the IC at time of the submission of this call. A new return value is generated for each call to this method. +[#ic-http_request] +=== IC method `http_request` + +This method makes an HTTP request. + +// TODO: Flesh out the documentation + [#ic-provisional_create_canister_with_cycles] === IC method `provisional_create_canister_with_cycles` From da47e4a880cbb5b39ff3598b7c8c0d31bdf0e774 Mon Sep 17 00:00:00 2001 From: Islam El-Ashi Date: Thu, 27 Jan 2022 09:15:26 +0100 Subject: [PATCH 2/9] . --- spec/index.adoc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/index.adoc b/spec/index.adoc index d9d56b69b..8136bc5fb 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1659,7 +1659,13 @@ This method takes no input and returns 32 pseudo-random bytes to the caller. The This method makes an HTTP request. -// TODO: Flesh out the documentation +// TODO: Flesh out the documentation. Some information you might want to include: +// * Can you make any HTTP request? What are the constraints? +// +// * Is there a cost associated with this call? If we know we'll charge for it, but don't know +// the exact amount, adding a placeholder would be sufficient in the first draft. +// +// * The `func` in the `transform` parameter must be a function exported by the canister itself (and not another canister). [#ic-provisional_create_canister_with_cycles] === IC method `provisional_create_canister_with_cycles` From 2c84d302a749443c59682da4ae1163d8b86e89e1 Mon Sep 17 00:00:00 2001 From: Yotam Harchol Date: Thu, 27 Jan 2022 15:23:31 +0100 Subject: [PATCH 3/9] Initial text for http_request --- spec/index.adoc | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/spec/index.adoc b/spec/index.adoc index 8136bc5fb..7fff9b8b2 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1657,7 +1657,34 @@ This method takes no input and returns 32 pseudo-random bytes to the caller. The [#ic-http_request] === IC method `http_request` -This method makes an HTTP request. +This method makes an HTTP request to a given URL and returns the HTTP response, possibly after a transformation. + +For security, the request is executed by multiple IC replicas (usually the entire subnet) and therefore must be _idempotent_. This means that the request must not change state at the remote server, or that the remote server has means to identify duplicated requests and return the same response to all of them. + +The responses for all the requests must be, eventually, identical. However, it is possible that a web service would return slightly different responses for the same idempotent request. For example, it may include some unique identification or a timestamp that would vary across responses. + +For this reason, the calling canister can supply a transformation function, which is used by the IC to let the canister sanitize the responses from such unique values, and therefore eventually agree on an identical response. The transformation function is executed separately by each replica on the corresponding response it has received for the request. The final response will only be available to the calling canister if enough replicas have agreed on an identical response. + +Currently, only GET method is supported for HTTP requests. + +For security reasons, only HTTPS connections are allowed (URLs must start with `https://`). The IC uses industry-standard root CA lists to validate certificates of remote web servers. + +The cost of each call to the `http_request` method is `TBD` (the `transform` should also be charged). + +The follwing paramters should be supplied for the call: + +- `url` - the requested URL +- `method` - currently only GET is supported +- `headers` - list of HTTP request headers and their corresponding values +- `transform` - an optional function that transforms raw responses to sanitized responses. This must be a function exported by the canister itself (and not another canister). + +The returned response (and the response provided to the `transform` function, if specified), contains the following fields: + +- `status` - the response status (e.g., 200, 404) +- `headers` - list of HTTP response headers and their corresponding values +- `body` - the response's body + +The `transform` function may, for example, transform the body in any way, add or remove headers, modify headers, etc. // TODO: Flesh out the documentation. Some information you might want to include: // * Can you make any HTTP request? What are the constraints? From 363f3f8d5d4ab929803f6ad7f27f70e6515ecc90 Mon Sep 17 00:00:00 2001 From: Yotam Harchol Date: Fri, 28 Jan 2022 12:46:09 +0100 Subject: [PATCH 4/9] Adding info on maximal sizes --- spec/index.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/index.adoc b/spec/index.adoc index 7fff9b8b2..64f2ac22c 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1671,6 +1671,11 @@ For security reasons, only HTTPS connections are allowed (URLs must start with ` The cost of each call to the `http_request` method is `TBD` (the `transform` should also be charged). +The maximal size of a request URL is 2048 bytes. + +The maximal size of a response is `TBD`. If a response is larger than this size, only the first `TBD` bytes will be returned. This size limit also applies to the value returned by the `transform` function. +// NOTE: the size above should be up to 2MB which is the maximal size of a message in the consensus queue + The follwing paramters should be supplied for the call: - `url` - the requested URL From 8fff092eba1e59944fe647c75df388efa89d9657 Mon Sep 17 00:00:00 2001 From: Yotam Harchol Date: Mon, 31 Jan 2022 13:53:52 +0100 Subject: [PATCH 5/9] Addressing review comments --- spec/ic.did | 1 + spec/index.adoc | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/ic.did b/spec/ic.did index e608cfb7d..f9f3f0908 100644 --- a/spec/ic.did +++ b/spec/ic.did @@ -28,6 +28,7 @@ type http_request_error = variant { no_consensus; timeout; bad_tls; + invalid_url; }; service ic : { diff --git a/spec/index.adoc b/spec/index.adoc index 64f2ac22c..a2181de5e 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1659,7 +1659,7 @@ This method takes no input and returns 32 pseudo-random bytes to the caller. The This method makes an HTTP request to a given URL and returns the HTTP response, possibly after a transformation. -For security, the request is executed by multiple IC replicas (usually the entire subnet) and therefore must be _idempotent_. This means that the request must not change state at the remote server, or that the remote server has means to identify duplicated requests and return the same response to all of them. +For security, the request is executed by multiple IC replicas (usually the entire subnet) and therefore must be _idempotent_. This means that the request must not change state at the remote server, or that the remote server has means to identify duplicated requests. The responses for all the requests must be, eventually, identical. However, it is possible that a web service would return slightly different responses for the same idempotent request. For example, it may include some unique identification or a timestamp that would vary across responses. @@ -1676,7 +1676,7 @@ The maximal size of a request URL is 2048 bytes. The maximal size of a response is `TBD`. If a response is larger than this size, only the first `TBD` bytes will be returned. This size limit also applies to the value returned by the `transform` function. // NOTE: the size above should be up to 2MB which is the maximal size of a message in the consensus queue -The follwing paramters should be supplied for the call: +The following parameters should be supplied for the call: - `url` - the requested URL - `method` - currently only GET is supported From eff6236638d3e852510d702df741d4d09aa21ffc Mon Sep 17 00:00:00 2001 From: Yotam Harchol Date: Wed, 2 Feb 2022 10:30:06 +0100 Subject: [PATCH 6/9] Adding more error codes --- spec/ic.did | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/ic.did b/spec/ic.did index f9f3f0908..3b631e86b 100644 --- a/spec/ic.did +++ b/spec/ic.did @@ -29,6 +29,10 @@ type http_request_error = variant { timeout; bad_tls; invalid_url; + transform_error; + dns_error; + unreachable; + conn_timeout; }; service ic : { From 3cfd100a4a91ddfdba7f86017c0b95eb5ca93322 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 21 Feb 2022 13:35:33 +0100 Subject: [PATCH 7/9] Finalise IC-530 spec --- spec/ic.did | 2 +- spec/index.adoc | 28 ++++++++-------------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/spec/ic.did b/spec/ic.did index 3b631e86b..e43304b75 100644 --- a/spec/ic.did +++ b/spec/ic.did @@ -16,7 +16,7 @@ type definite_canister_settings = record { freezing_threshold : nat; }; -type http_header = record { 0: text; 1: text }; +type http_header = record { name: text; value: text }; type http_response = record { status: nat; diff --git a/spec/index.adoc b/spec/index.adoc index a2181de5e..6c3176015 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1530,7 +1530,6 @@ The binary encoding of arguments and results are as per Candid specification. Before deploying a canister, the administrator of the canister first has to register it with the IC, to get a canister id (with an empty canister behind it), and then separately install the code. The optional `settings` parameter can be used to set the following settings: - * `controllers` (`vec principal`) + A list of principals. Must be between 0 and 10 in size. This value is assigned to the _controllers_ attribute of the canister. @@ -1659,31 +1658,28 @@ This method takes no input and returns 32 pseudo-random bytes to the caller. The This method makes an HTTP request to a given URL and returns the HTTP response, possibly after a transformation. -For security, the request is executed by multiple IC replicas (usually the entire subnet) and therefore must be _idempotent_. This means that the request must not change state at the remote server, or that the remote server has means to identify duplicated requests. +The request must be _idempotent_, meaning that it must not change the state at the remote server, or the remote server has the means to identify duplicated requests. -The responses for all the requests must be, eventually, identical. However, it is possible that a web service would return slightly different responses for the same idempotent request. For example, it may include some unique identification or a timestamp that would vary across responses. +The responses for all identical requests must match too. However, a web service would return slightly different responses for identical idempotent requests. For example, it may include some unique identification or a timestamp that would vary across responses. -For this reason, the calling canister can supply a transformation function, which is used by the IC to let the canister sanitize the responses from such unique values, and therefore eventually agree on an identical response. The transformation function is executed separately by each replica on the corresponding response it has received for the request. The final response will only be available to the calling canister if enough replicas have agreed on an identical response. +For this reason, the calling canister can supply a transformation function, which the IC uses to let the canister sanitize the responses from such unique values. The transformation function is executed separately on the corresponding response received for a request. The final response will only be available to the calling canister. -Currently, only GET method is supported for HTTP requests. +Currently, only the `GET` method is supported for HTTP requests. For security reasons, only HTTPS connections are allowed (URLs must start with `https://`). The IC uses industry-standard root CA lists to validate certificates of remote web servers. -The cost of each call to the `http_request` method is `TBD` (the `transform` should also be charged). - The maximal size of a request URL is 2048 bytes. -The maximal size of a response is `TBD`. If a response is larger than this size, only the first `TBD` bytes will be returned. This size limit also applies to the value returned by the `transform` function. -// NOTE: the size above should be up to 2MB which is the maximal size of a message in the consensus queue +The maximal size of a response is `2MiB`. Therefore, only the first `2MiB` will be returned if a response is larger than this size. This size limit also applies to the value returned by the `transform` function. The following parameters should be supplied for the call: - `url` - the requested URL -- `method` - currently only GET is supported +- `method` - currently, only GET is supported - `headers` - list of HTTP request headers and their corresponding values -- `transform` - an optional function that transforms raw responses to sanitized responses. This must be a function exported by the canister itself (and not another canister). +- `transform` - an optional function that transforms raw responses to sanitized responses. If provided, the calling canister itself must export this function. -The returned response (and the response provided to the `transform` function, if specified), contains the following fields: +The returned response (and the response provided to the `transform` function, if specified) contains the following fields: - `status` - the response status (e.g., 200, 404) - `headers` - list of HTTP response headers and their corresponding values @@ -1691,14 +1687,6 @@ The returned response (and the response provided to the `transform` function, if The `transform` function may, for example, transform the body in any way, add or remove headers, modify headers, etc. -// TODO: Flesh out the documentation. Some information you might want to include: -// * Can you make any HTTP request? What are the constraints? -// -// * Is there a cost associated with this call? If we know we'll charge for it, but don't know -// the exact amount, adding a placeholder would be sufficient in the first draft. -// -// * The `func` in the `transform` parameter must be a function exported by the canister itself (and not another canister). - [#ic-provisional_create_canister_with_cycles] === IC method `provisional_create_canister_with_cycles` From fb84ad7a0c09fac70721d0d494a80568c0908d8b Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 21 Feb 2022 14:10:07 +0100 Subject: [PATCH 8/9] . --- spec/index.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/index.adoc b/spec/index.adoc index 6c3176015..b17e26f0d 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1530,6 +1530,7 @@ The binary encoding of arguments and results are as per Candid specification. Before deploying a canister, the administrator of the canister first has to register it with the IC, to get a canister id (with an empty canister behind it), and then separately install the code. The optional `settings` parameter can be used to set the following settings: + * `controllers` (`vec principal`) + A list of principals. Must be between 0 and 10 in size. This value is assigned to the _controllers_ attribute of the canister. @@ -1660,7 +1661,7 @@ This method makes an HTTP request to a given URL and returns the HTTP response, The request must be _idempotent_, meaning that it must not change the state at the remote server, or the remote server has the means to identify duplicated requests. -The responses for all identical requests must match too. However, a web service would return slightly different responses for identical idempotent requests. For example, it may include some unique identification or a timestamp that would vary across responses. +The responses for all identical requests must match too. However, a web service could return slightly different responses for identical idempotent requests. For example, it may include some unique identification or a timestamp that would vary across responses. For this reason, the calling canister can supply a transformation function, which the IC uses to let the canister sanitize the responses from such unique values. The transformation function is executed separately on the corresponding response received for a request. The final response will only be available to the calling canister. From f459742f7c6bc271ca2549455d6f693168dbfb99 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 22 Feb 2022 14:05:53 +0100 Subject: [PATCH 9/9] Jens review --- spec/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/index.adoc b/spec/index.adoc index b17e26f0d..d83c6ffaf 100644 --- a/spec/index.adoc +++ b/spec/index.adoc @@ -1659,7 +1659,7 @@ This method takes no input and returns 32 pseudo-random bytes to the caller. The This method makes an HTTP request to a given URL and returns the HTTP response, possibly after a transformation. -The request must be _idempotent_, meaning that it must not change the state at the remote server, or the remote server has the means to identify duplicated requests. +The canister should aim to issue _idempotent_ requests, meaning that it must not change the state at the remote server, or the remote server has the means to identify duplicated requests. Otherwise, the risk of failure increases. The responses for all identical requests must match too. However, a web service could return slightly different responses for identical idempotent requests. For example, it may include some unique identification or a timestamp that would vary across responses.