From 9af7c5f92abfa876ddc258b9ff5a72fe41aaf003 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Wed, 10 Apr 2019 09:42:29 +0700 Subject: [PATCH 01/14] Copy TAKE_PTR from systemd Signed-off-by: Arnaud Rebillout --- src/util.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/util.h b/src/util.h index dc9d03e4..68299a5b 100644 --- a/src/util.h +++ b/src/util.h @@ -765,6 +765,15 @@ DEFINE_TRIVIAL_CLEANUP_FUNC(char*, unlink_and_free); int free_and_strdup(char **p, const char *s); +/* Takes inspiration from Rusts's Option::take() method: reads and returns a pointer, but at the same time resets it to + * NULL. See: https://doc.rust-lang.org/std/option/enum.Option.html#method.take */ +#define TAKE_PTR(ptr) \ + ({ \ + typeof(ptr) _ptr_ = (ptr); \ + (ptr) = NULL; \ + _ptr_; \ + }) + /* A check against a list of errors commonly used to indicate that a syscall/ioctl/other kernel operation we request is * not supported locally. We maintain a generic list for this here, instead of adjusting the possible error codes to * exactly what the calls might return for the simple reasons that due to FUSE and many differing in-kernel From 74e294a7345f1208e4f203769552583756cc9eac Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 00:38:06 +0700 Subject: [PATCH 02/14] casync-http: Rename ARG_PROTOCOL_* to PROTOCOL_*, add a typedef This is to prepare the next commit, where we will use the protocol enum for more than just the protocol passed in arguments, and the arg_ prefix won't make sense anymore. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 43809c3a..f03a6f6e 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -16,13 +16,15 @@ static volatile sig_atomic_t quit = false; static bool arg_verbose = false; static curl_off_t arg_rate_limit_bps = 0; -static enum { - ARG_PROTOCOL_HTTP, - ARG_PROTOCOL_FTP, - ARG_PROTOCOL_HTTPS, - ARG_PROTOCOL_SFTP, - _ARG_PROTOCOL_INVALID = -1, -} arg_protocol = _ARG_PROTOCOL_INVALID; +typedef enum Protocol { + PROTOCOL_HTTP, + PROTOCOL_FTP, + PROTOCOL_HTTPS, + PROTOCOL_SFTP, + _PROTOCOL_INVALID = -1, +} Protocol; + +static Protocol arg_protocol = _PROTOCOL_INVALID; typedef enum ProcessUntil { PROCESS_UNTIL_WRITTEN, @@ -326,7 +328,7 @@ static int acquire_file(CaRemote *rr, return -EIO; } - if (IN_SET(arg_protocol, ARG_PROTOCOL_HTTP, ARG_PROTOCOL_HTTPS) && protocol_status != 200) { + if (IN_SET(arg_protocol, PROTOCOL_HTTP, PROTOCOL_HTTPS) && protocol_status != 200) { char *m; if (arg_verbose) @@ -340,7 +342,7 @@ static int acquire_file(CaRemote *rr, return 0; - } else if (arg_protocol == ARG_PROTOCOL_FTP && (protocol_status < 200 || protocol_status > 299)) { + } else if (arg_protocol == PROTOCOL_FTP && (protocol_status < 200 || protocol_status > 299)) { char *m; if (arg_verbose) @@ -352,7 +354,7 @@ static int acquire_file(CaRemote *rr, (void) ca_remote_abort(rr, EBADR, m); free(m); return 0; - } else if (arg_protocol == ARG_PROTOCOL_SFTP && (protocol_status != 0)) { + } else if (arg_protocol == PROTOCOL_SFTP && (protocol_status != 0)) { char *m; if (arg_verbose) @@ -436,14 +438,14 @@ static int run(int argc, char *argv[]) { goto finish; } - if (curl_easy_setopt(curl, CURLOPT_PROTOCOLS, arg_protocol == ARG_PROTOCOL_FTP ? CURLPROTO_FTP : - arg_protocol == ARG_PROTOCOL_SFTP? CURLPROTO_SFTP: CURLPROTO_HTTP|CURLPROTO_HTTPS) != CURLE_OK) { + if (curl_easy_setopt(curl, CURLOPT_PROTOCOLS, arg_protocol == PROTOCOL_FTP ? CURLPROTO_FTP : + arg_protocol == PROTOCOL_SFTP? CURLPROTO_SFTP: CURLPROTO_HTTP|CURLPROTO_HTTPS) != CURLE_OK) { log_error("Failed to limit protocols to HTTP/HTTPS/FTP/SFTP."); r = -EIO; goto finish; } - if (arg_protocol == ARG_PROTOCOL_SFTP) { + if (arg_protocol == PROTOCOL_SFTP) { /* activate the ssh agent. For this to work you need to have ssh-agent running (type set | grep SSH_AGENT to check) */ if (curl_easy_setopt(curl, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_AGENT) != CURLE_OK) @@ -587,9 +589,9 @@ static int run(int argc, char *argv[]) { if (r < 0) goto finish; - if ((IN_SET(arg_protocol, ARG_PROTOCOL_HTTP, ARG_PROTOCOL_HTTPS) && protocol_status == 200) || - (arg_protocol == ARG_PROTOCOL_FTP && (protocol_status >= 200 && protocol_status <= 299))|| - (arg_protocol == ARG_PROTOCOL_SFTP && (protocol_status == 0))) { + if ((IN_SET(arg_protocol, PROTOCOL_HTTP, PROTOCOL_HTTPS) && protocol_status == 200) || + (arg_protocol == PROTOCOL_FTP && (protocol_status >= 200 && protocol_status <= 299))|| + (arg_protocol == PROTOCOL_SFTP && (protocol_status == 0))) { r = ca_remote_put_chunk(rr, &id, CA_CHUNK_COMPRESSED, realloc_buffer_data(&chunk_buffer), realloc_buffer_size(&chunk_buffer)); if (r < 0) { @@ -648,13 +650,13 @@ static int parse_argv(int argc, char *argv[]) { assert(argv); if (strstr(argv[0], "https")) - arg_protocol = ARG_PROTOCOL_HTTPS; + arg_protocol = PROTOCOL_HTTPS; else if (strstr(argv[0], "http")) - arg_protocol = ARG_PROTOCOL_HTTP; + arg_protocol = PROTOCOL_HTTP; else if (strstr(argv[0], "sftp")) - arg_protocol = ARG_PROTOCOL_SFTP; + arg_protocol = PROTOCOL_SFTP; else if (strstr(argv[0], "ftp")) - arg_protocol = ARG_PROTOCOL_FTP; + arg_protocol = PROTOCOL_FTP; else { log_error("Failed to determine set of protocols to use, refusing."); return -EINVAL; From 10c158c0fb80747ef150062e4b31c6a98850256a Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 00:54:30 +0700 Subject: [PATCH 03/14] casync-http: Add protocol helpers to factorize code This commits brings in two helpers: - protocol_str() to convert an enum protocol to a string, which is useful mainly for logs. - protocol_status_ok() as a unique place to check if the protocol status that we get from libcurl means OK or KO. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 90 +++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index f03a6f6e..dc753c5e 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -35,6 +35,47 @@ typedef enum ProcessUntil { PROCESS_UNTIL_FINISHED, } ProcessUntil; +/* + * protocol helpers + */ + +static const char *protocol_str(Protocol protocol) { + switch (protocol) { + case PROTOCOL_HTTP: + return "HTTP"; + case PROTOCOL_FTP: + return "FTP"; + case PROTOCOL_HTTPS: + return "HTTPS"; + case PROTOCOL_SFTP: + return "SFTP"; + default: + assert_not_reached("Unknown protocol"); + } +} + +static bool protocol_status_ok(Protocol protocol, long protocol_status) { + switch (protocol) { + case PROTOCOL_HTTP: + case PROTOCOL_HTTPS: + if (protocol_status == 200) + return true; + break; + case PROTOCOL_FTP: + if (protocol_status >= 200 && protocol_status <= 299) + return true; + break; + case PROTOCOL_SFTP: + if (protocol_status == 0) + return true; + break; + default: + assert_not_reached("Unknown protocol"); + break; + } + return false; +} + static CURLcode robust_curl_easy_perform(CURL *curl) { uint64_t sleep_base_usec = 100 * 1000; unsigned trial = 1; @@ -328,42 +369,24 @@ static int acquire_file(CaRemote *rr, return -EIO; } - if (IN_SET(arg_protocol, PROTOCOL_HTTP, PROTOCOL_HTTPS) && protocol_status != 200) { + if (!protocol_status_ok(arg_protocol, protocol_status)) { char *m; + int abort_code; if (arg_verbose) - log_error("HTTP server failure %li while requesting %s.", protocol_status, url); + log_error("%s server failure %li while requesting %s", + protocol_str(arg_protocol), protocol_status, url); - if (asprintf(&m, "HTTP request on %s failed with status %li", url, protocol_status) < 0) + if (asprintf(&m, "%s request on %s failed with status %li", + protocol_str(arg_protocol), url, protocol_status) < 0) return log_oom(); - (void) ca_remote_abort(rr, protocol_status == 404 ? ENOMEDIUM : EBADR, m); - free(m); - - return 0; - - } else if (arg_protocol == PROTOCOL_FTP && (protocol_status < 200 || protocol_status > 299)) { - char *m; - - if (arg_verbose) - log_error("FTP server failure %li while requesting %s.", protocol_status, url); - - if (asprintf(&m, "FTP request on %s failed with status %li", url, protocol_status) < 0) - return log_oom(); - - (void) ca_remote_abort(rr, EBADR, m); - free(m); - return 0; - } else if (arg_protocol == PROTOCOL_SFTP && (protocol_status != 0)) { - char *m; - - if (arg_verbose) - log_error("SFTP server failure %li while requesting %s.", protocol_status, url); - - if (asprintf(&m, "SFTP request on %s failed with status %li", url, protocol_status) < 0) - return log_oom(); + if (IN_SET(arg_protocol, PROTOCOL_HTTP, PROTOCOL_HTTPS) && protocol_status == 404) + abort_code = ENOMEDIUM; + else + abort_code = EBADR; - (void) ca_remote_abort(rr, EBADR, m); + (void) ca_remote_abort(rr, abort_code, m); free(m); return 0; } @@ -589,10 +612,7 @@ static int run(int argc, char *argv[]) { if (r < 0) goto finish; - if ((IN_SET(arg_protocol, PROTOCOL_HTTP, PROTOCOL_HTTPS) && protocol_status == 200) || - (arg_protocol == PROTOCOL_FTP && (protocol_status >= 200 && protocol_status <= 299))|| - (arg_protocol == PROTOCOL_SFTP && (protocol_status == 0))) { - + if (protocol_status_ok(arg_protocol, protocol_status)) { r = ca_remote_put_chunk(rr, &id, CA_CHUNK_COMPRESSED, realloc_buffer_data(&chunk_buffer), realloc_buffer_size(&chunk_buffer)); if (r < 0) { log_error_errno(r, "Failed to write chunk: %m"); @@ -601,7 +621,9 @@ static int run(int argc, char *argv[]) { } else { if (arg_verbose) - log_error("HTTP/FTP/SFTP server failure %li while requesting %s.", protocol_status, url_buffer); + log_error("%s server failure %ld while requesting %s", + protocol_str(arg_protocol), protocol_status, + url_buffer); r = ca_remote_put_missing(rr, &id); if (r < 0) { From d7b7bb279de865f32c153eed4c35a3ccde2568d4 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 14:14:52 +0700 Subject: [PATCH 04/14] casync-http: Use automatic pointer in acquire_file() Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index dc753c5e..b1b9f657 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -370,7 +370,7 @@ static int acquire_file(CaRemote *rr, } if (!protocol_status_ok(arg_protocol, protocol_status)) { - char *m; + _cleanup_free_ char *m = NULL; int abort_code; if (arg_verbose) @@ -387,7 +387,6 @@ static int acquire_file(CaRemote *rr, abort_code = EBADR; (void) ca_remote_abort(rr, abort_code, m); - free(m); return 0; } From ed01920db5f7fd6d6f85bf30c62eb0bcccaf24b7 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 00:57:17 +0700 Subject: [PATCH 05/14] casync-http: Fix handling of ca_remote_has_unwritten() return value It seems to me that the condition PROCESS_UNTIL_WRITTEN is reached when there's no more data to write, hence ca_remote_has_unwritten() returns 0. And it also seems that this is could be copy/paste mistake, as all the code above is similar, but the condition matches the function we call, ie: - PROCESS_UNTIL_CAN_PUT_CHUNK > ca_remote_can_put_chunk - PROCESS_UNTIL_CAN_PUT_INDEX > ca_remote_can_put_index - PROCESS_UNTIL_CAN_PUT_ARCHIVE > ca_remote_can_put_archive - PROCESS_UNTIL_HAVE_REQUEST > ca_remote_has_pending_requests But here, the function returns the opposite of what we want: - PROCESS_UNTIL_WRITTEN > ca_remote_has_unwritten Note that I didn't observe any bug due to that, and the test suite succeeds before and after this patch. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/casync-http.c b/src/casync-http.c index b1b9f657..54b17bc9 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -178,7 +178,7 @@ static int process_remote(CaRemote *rr, ProcessUntil until) { return r; if (r < 0) return log_error_errno(r, "Failed to determine whether there's more data to write."); - if (r > 0) + if (r == 0) return 0; break; From b88e11533ed1ca6ecd86608c90d0e9416112027b Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 00:59:16 +0700 Subject: [PATCH 06/14] casync-http: Dont forget to call PROCESS_UNTIL_WRITTEN While working on this code, I stumbled on cases where casync got stuck because we forgot to call PROCESS_UNTIL_WRITTEN here. Well, TBH as long as we download chunks, we're fine because we end up calling PROCESS_UNTIL_WRITTEN afterwards. But in any case, it seems more correct to sync after downloading these files, and it doesn't hurt. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/casync-http.c b/src/casync-http.c index 54b17bc9..0c945f07 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -227,6 +227,10 @@ static size_t write_index(const void *buffer, size_t size, size_t nmemb, void *u return 0; } + r = process_remote(rr, PROCESS_UNTIL_WRITTEN); + if (r < 0) + return r; + return product; } @@ -243,6 +247,10 @@ static int write_index_eof(CaRemote *rr) { if (r < 0) return log_error_errno(r, "Failed to put index EOF: %m"); + r = process_remote(rr, PROCESS_UNTIL_WRITTEN); + if (r < 0) + return r; + return 0; } @@ -263,6 +271,10 @@ static size_t write_archive(const void *buffer, size_t size, size_t nmemb, void return 0; } + r = process_remote(rr, PROCESS_UNTIL_WRITTEN); + if (r < 0) + return r; + return product; } @@ -279,6 +291,10 @@ static int write_archive_eof(CaRemote *rr) { if (r < 0) return log_error_errno(r, "Failed to put archive EOF: %m"); + r = process_remote(rr, PROCESS_UNTIL_WRITTEN); + if (r < 0) + return r; + return 0; } From 669ab872f1dde8d821891db578839e2d91271162 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 15:23:41 +0700 Subject: [PATCH 07/14] casync-http: Factorize curl handle work The way we use curl handle is that we create it once, and then re-use it again and again, as it's more efficient than re-allocating a new one for each request. By looking at the code closely, it turns out that the setup of the curl handle needs to be done only once, then afterwards we only need to change the URL in order to re-use the handle. So this commit brings two helper functions to reflect that: - make_curl_easy_handle() does the init work and set all the options for the handle. - configure_curl_easy_handle() does the things that are needed in order to re-use the handle. In effect, it only sets the URL. Additionally, this commit introduces curl_easy_cleanupp, in order to use automatic pointers. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 226 +++++++++++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 94 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 0c945f07..74f526e9 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -76,6 +76,107 @@ static bool protocol_status_ok(Protocol protocol, long protocol_status) { return false; } +/* + * curl helpers + */ + +DEFINE_TRIVIAL_CLEANUP_FUNC(CURL*, curl_easy_cleanup); + +static inline const char *get_curl_effective_url(CURL *handle) { + CURLcode c; + char *effective_url; + + c = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &effective_url); + if (c != CURLE_OK) { + log_error("Failed to get CURL effective URL."); + return NULL; + } + + return effective_url; +} + +static int configure_curl_easy_handle(CURL *handle, const char *url) { + assert(handle); + assert(url); + + if (curl_easy_setopt(handle, CURLOPT_URL, url) != CURLE_OK) { + log_error("Failed to set CURL URL to: %s", url); + return -EIO; + } + + return 0; +} + +typedef size_t (*ca_curl_write_callback_t)(const void *, size_t, size_t, void *); + +static int make_curl_easy_handle(CURL **ret, + ca_curl_write_callback_t write_callback, + void *write_data, void *private) { + _cleanup_(curl_easy_cleanupp) CURL *h = NULL; + + assert(ret); + assert(write_callback); + assert(write_data); + /* private is optional and can be null */ + + h = curl_easy_init(); + if (!h) + return log_oom(); + + if (curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) { + log_error("Failed to turn on location following."); + return -EIO; + } + + if (curl_easy_setopt(h, CURLOPT_PROTOCOLS, arg_protocol == PROTOCOL_FTP ? CURLPROTO_FTP : + arg_protocol == PROTOCOL_SFTP ? CURLPROTO_SFTP : + CURLPROTO_HTTP | CURLPROTO_HTTPS) != CURLE_OK) { + log_error("Failed to limit protocols to HTTP/HTTPS/FTP/SFTP."); + return -EIO; + } + + if (arg_protocol == PROTOCOL_SFTP) { + /* activate the ssh agent. For this to work you need + to have ssh-agent running (type set | grep SSH_AGENT to check) */ + if (curl_easy_setopt(h, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_AGENT) != CURLE_OK) + log_error("Failed to turn on ssh agent support, ignoring."); + } + + if (arg_rate_limit_bps > 0) { + if (curl_easy_setopt(h, CURLOPT_MAX_SEND_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { + log_error("Failed to set CURL send speed limit."); + return -EIO; + } + + if (curl_easy_setopt(h, CURLOPT_MAX_RECV_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { + log_error("Failed to set CURL receive speed limit."); + return -EIO; + } + } + + if (curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, write_callback) != CURLE_OK) { + log_error("Failed to set CURL callback function."); + return -EIO; + } + + if (curl_easy_setopt(h, CURLOPT_WRITEDATA, write_data) != CURLE_OK) { + log_error("Failed to set CURL callback data."); + return -EIO; + } + + if (private) { + if (curl_easy_setopt(h, CURLOPT_PRIVATE, private) != CURLE_OK) { + log_error("Failed to set CURL private data."); + return -EIO; + } + } + + /* (void) curl_easy_setopt(h, CURLOPT_VERBOSE, 1L); */ + + *ret = TAKE_PTR(h); + return 0; +} + static CURLcode robust_curl_easy_perform(CURL *curl) { uint64_t sleep_base_usec = 100 * 1000; unsigned trial = 1; @@ -347,40 +448,21 @@ static char *chunk_url(const char *store_url, const CaChunkID *id) { return buffer; } -static int acquire_file(CaRemote *rr, - CURL *curl, - const char *url, - size_t (*callback)(const void *p, size_t size, size_t nmemb, void *userdata)) { - +static int acquire_file(CaRemote *rr, CURL *handle) { long protocol_status; + const char *url; - assert(curl); + url = get_curl_effective_url(handle); assert(url); - assert(callback); - - if (curl_easy_setopt(curl, CURLOPT_URL, url) != CURLE_OK) { - log_error("Failed to set CURL URL to: %s", url); - return -EIO; - } - - if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback) != CURLE_OK) { - log_error("Failed to set CURL callback function."); - return -EIO; - } - - if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, rr) != CURLE_OK) { - log_error("Failed to set CURL private data."); - return -EIO; - } log_debug("Acquiring %s...", url); - if (robust_curl_easy_perform(curl) != CURLE_OK) { + if (robust_curl_easy_perform(handle) != CURLE_OK) { log_error("Failed to acquire %s", url); return -EIO; } - if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) { + if (curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) { log_error("Failed to query response code"); return -EIO; } @@ -464,50 +546,18 @@ static int run(int argc, char *argv[]) { goto finish; } - curl = curl_easy_init(); - if (!curl) { - r = log_oom(); - goto finish; - } - - if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) { - log_error("Failed to turn on location following."); - r = -EIO; - goto finish; - } - - if (curl_easy_setopt(curl, CURLOPT_PROTOCOLS, arg_protocol == PROTOCOL_FTP ? CURLPROTO_FTP : - arg_protocol == PROTOCOL_SFTP? CURLPROTO_SFTP: CURLPROTO_HTTP|CURLPROTO_HTTPS) != CURLE_OK) { - log_error("Failed to limit protocols to HTTP/HTTPS/FTP/SFTP."); - r = -EIO; - goto finish; - } - - if (arg_protocol == PROTOCOL_SFTP) { - /* activate the ssh agent. For this to work you need - to have ssh-agent running (type set | grep SSH_AGENT to check) */ - if (curl_easy_setopt(curl, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_AGENT) != CURLE_OK) - log_error("Failed to turn on ssh agent support, ignoring."); - } + if (archive_url) { + _cleanup_(curl_easy_cleanupp) CURL *handle = NULL; - if (arg_rate_limit_bps > 0) { - if (curl_easy_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL send speed limit."); - r = -EIO; + r = make_curl_easy_handle(&handle, write_archive, rr, NULL); + if (r < 0) goto finish; - } - if (curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL receive speed limit."); - r = -EIO; + r = configure_curl_easy_handle(handle, archive_url); + if (r < 0) goto finish; - } - } - - /* (void) curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); */ - if (archive_url) { - r = acquire_file(rr, curl, archive_url, write_archive); + r = acquire_file(rr, handle); if (r < 0) goto finish; if (r == 0) @@ -519,7 +569,17 @@ static int run(int argc, char *argv[]) { } if (index_url) { - r = acquire_file(rr, curl, index_url, write_index); + _cleanup_(curl_easy_cleanupp) CURL *handle = NULL; + + r = make_curl_easy_handle(&handle, write_index, rr, NULL); + if (r < 0) + goto finish; + + r = configure_curl_easy_handle(handle, index_url); + if (r < 0) + goto finish; + + r = acquire_file(rr, handle); if (r < 0) goto finish; if (r == 0) @@ -543,6 +603,12 @@ static int run(int argc, char *argv[]) { if (n_stores == 0) /* No stores? Then we did all we could do */ break; + if (!curl) { + r = make_curl_easy_handle(&curl, write_chunk, &chunk_buffer, NULL); + if (r < 0) + goto finish; + } + r = process_remote(rr, PROCESS_UNTIL_HAVE_REQUEST); if (r == -EPIPE) { r = 0; @@ -573,37 +639,9 @@ static int run(int argc, char *argv[]) { goto finish; } - if (curl_easy_setopt(curl, CURLOPT_URL, url_buffer) != CURLE_OK) { - log_error("Failed to set CURL URL to: %s", index_url); - r = -EIO; - goto finish; - } - - if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_chunk) != CURLE_OK) { - log_error("Failed to set CURL callback function."); - r = -EIO; - goto finish; - } - - if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk_buffer) != CURLE_OK) { - log_error("Failed to set CURL private data."); - r = -EIO; + r = configure_curl_easy_handle(curl, url_buffer); + if (r < 0) goto finish; - } - - if (arg_rate_limit_bps > 0) { - if (curl_easy_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL send speed limit."); - r = -EIO; - goto finish; - } - - if (curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL receive speed limit."); - r = -EIO; - goto finish; - } - } log_debug("Acquiring %s...", url_buffer); From db2559488a13963220b3c231b2ae68df6d539a89 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 15:49:56 +0700 Subject: [PATCH 08/14] casync-http: Introduce macros to set curl options These macros aim to make setting curl options easier. CURL_SETOPT_EASY() sets the option, and on failure it outputs a generic error message with the name of the option that failed, and returns -EIO. The CURL_SETOPT_EASY_CANFAIL() variant does not return, it only outputs an error message. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 77 +++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 74f526e9..ff0aed26 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -82,13 +82,32 @@ static bool protocol_status_ok(Protocol protocol, long protocol_status) { DEFINE_TRIVIAL_CLEANUP_FUNC(CURL*, curl_easy_cleanup); +#define log_error_curle(code, fmt, ...) \ + log_error_errno(-EIO, fmt ": %s", ##__VA_ARGS__, curl_easy_strerror(code)) + +#define CURL_SETOPT_EASY(handle, option, value) \ + ({ \ + CURLcode _c; \ + _c = curl_easy_setopt(handle, option, (value)); \ + if (_c != CURLE_OK) \ + return log_error_curle(_c, "Failed to set " #option); \ + }) + +#define CURL_SETOPT_EASY_CANFAIL(handle, option, value) \ + ({ \ + CURLcode _c; \ + _c = curl_easy_setopt(handle, option, (value)); \ + if (_c != CURLE_OK) \ + log_error_curle(_c, "Failed to set " #option); \ + }) + static inline const char *get_curl_effective_url(CURL *handle) { CURLcode c; char *effective_url; c = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &effective_url); if (c != CURLE_OK) { - log_error("Failed to get CURL effective URL."); + log_error_curle(c, "Failed to get CURLINFO_EFFECTIVE_URL"); return NULL; } @@ -99,10 +118,7 @@ static int configure_curl_easy_handle(CURL *handle, const char *url) { assert(handle); assert(url); - if (curl_easy_setopt(handle, CURLOPT_URL, url) != CURLE_OK) { - log_error("Failed to set CURL URL to: %s", url); - return -EIO; - } + CURL_SETOPT_EASY(handle, CURLOPT_URL, url); return 0; } @@ -123,55 +139,30 @@ static int make_curl_easy_handle(CURL **ret, if (!h) return log_oom(); - if (curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) { - log_error("Failed to turn on location following."); - return -EIO; - } - - if (curl_easy_setopt(h, CURLOPT_PROTOCOLS, arg_protocol == PROTOCOL_FTP ? CURLPROTO_FTP : - arg_protocol == PROTOCOL_SFTP ? CURLPROTO_SFTP : - CURLPROTO_HTTP | CURLPROTO_HTTPS) != CURLE_OK) { - log_error("Failed to limit protocols to HTTP/HTTPS/FTP/SFTP."); - return -EIO; - } + CURL_SETOPT_EASY(h, CURLOPT_FOLLOWLOCATION, 1L); + CURL_SETOPT_EASY(h, CURLOPT_PROTOCOLS, + arg_protocol == PROTOCOL_FTP ? CURLPROTO_FTP : + arg_protocol == PROTOCOL_SFTP ? CURLPROTO_SFTP : + CURLPROTO_HTTP | CURLPROTO_HTTPS); if (arg_protocol == PROTOCOL_SFTP) { /* activate the ssh agent. For this to work you need to have ssh-agent running (type set | grep SSH_AGENT to check) */ - if (curl_easy_setopt(h, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_AGENT) != CURLE_OK) - log_error("Failed to turn on ssh agent support, ignoring."); + CURL_SETOPT_EASY_CANFAIL(h, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_AGENT); } if (arg_rate_limit_bps > 0) { - if (curl_easy_setopt(h, CURLOPT_MAX_SEND_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL send speed limit."); - return -EIO; - } - - if (curl_easy_setopt(h, CURLOPT_MAX_RECV_SPEED_LARGE, arg_rate_limit_bps) != CURLE_OK) { - log_error("Failed to set CURL receive speed limit."); - return -EIO; - } + CURL_SETOPT_EASY(h, CURLOPT_MAX_SEND_SPEED_LARGE, arg_rate_limit_bps); + CURL_SETOPT_EASY(h, CURLOPT_MAX_RECV_SPEED_LARGE, arg_rate_limit_bps); } - if (curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, write_callback) != CURLE_OK) { - log_error("Failed to set CURL callback function."); - return -EIO; - } + CURL_SETOPT_EASY(h, CURLOPT_WRITEFUNCTION, write_callback); + CURL_SETOPT_EASY(h, CURLOPT_WRITEDATA, write_data); - if (curl_easy_setopt(h, CURLOPT_WRITEDATA, write_data) != CURLE_OK) { - log_error("Failed to set CURL callback data."); - return -EIO; - } - - if (private) { - if (curl_easy_setopt(h, CURLOPT_PRIVATE, private) != CURLE_OK) { - log_error("Failed to set CURL private data."); - return -EIO; - } - } + if (private) + CURL_SETOPT_EASY(h, CURLOPT_PRIVATE, private); - /* (void) curl_easy_setopt(h, CURLOPT_VERBOSE, 1L); */ + /* CURL_SETOPT_EASY(h, CURLOPT_VERBOSE, 1L); */ *ret = TAKE_PTR(h); return 0; From 1a29cdcf4c08f17b8a949bba3f1e56b886f460ad Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 15:53:07 +0700 Subject: [PATCH 09/14] casync-http: Use an automatic pointer for the chunks curl handle This removes the need for the 'finish' label, hence a bunch of goto go away. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 110 ++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 71 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index ff0aed26..10e49174 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -485,7 +485,7 @@ static int acquire_file(CaRemote *rr, CURL *handle) { static int run(int argc, char *argv[]) { const char *base_url, *archive_url, *index_url, *wstore_url; size_t n_stores = 0, current_store = 0; - CURL *curl = NULL; + _cleanup_(curl_easy_cleanupp) CURL *curl = NULL; _cleanup_(ca_remote_unrefp) CaRemote *rr = NULL; _cleanup_(realloc_buffer_free) ReallocBuffer chunk_buffer = {}; _cleanup_free_ char *url_buffer = NULL; @@ -517,46 +517,40 @@ static int run(int argc, char *argv[]) { } rr = ca_remote_new(); - if (!rr) { - r = log_oom(); - goto finish; - } + if (!rr) + return log_oom(); r = ca_remote_set_local_feature_flags(rr, (n_stores > 0 ? CA_PROTOCOL_READABLE_STORE : 0) | (index_url ? CA_PROTOCOL_READABLE_INDEX : 0) | (archive_url ? CA_PROTOCOL_READABLE_ARCHIVE : 0)); - if (r < 0) { - log_error("Failed to set feature flags: %m"); - goto finish; - } + if (r < 0) + return log_error_errno(r, "Failed to set feature flags: %m"); r = ca_remote_set_io_fds(rr, STDIN_FILENO, STDOUT_FILENO); - if (r < 0) { - log_error("Failed to set I/O file descriptors: %m"); - goto finish; - } + if (r < 0) + return log_error_errno(r, "Failed to set I/O file descriptors: %m"); if (archive_url) { _cleanup_(curl_easy_cleanupp) CURL *handle = NULL; r = make_curl_easy_handle(&handle, write_archive, rr, NULL); if (r < 0) - goto finish; + return r; r = configure_curl_easy_handle(handle, archive_url); if (r < 0) - goto finish; + return r; r = acquire_file(rr, handle); if (r < 0) - goto finish; + return r; if (r == 0) goto flush; r = write_archive_eof(rr); if (r < 0) - goto finish; + return r; } if (index_url) { @@ -564,21 +558,21 @@ static int run(int argc, char *argv[]) { r = make_curl_easy_handle(&handle, write_index, rr, NULL); if (r < 0) - goto finish; + return r; r = configure_curl_easy_handle(handle, index_url); if (r < 0) - goto finish; + return r; r = acquire_file(rr, handle); if (r < 0) - goto finish; + return r; if (r == 0) goto flush; r = write_index_eof(rr); if (r < 0) - goto finish; + return r; } for (;;) { @@ -587,8 +581,7 @@ static int run(int argc, char *argv[]) { if (quit) { log_info("Got exit signal, quitting."); - r = 0; - goto finish; + return 0; } if (n_stores == 0) /* No stores? Then we did all we could do */ @@ -597,24 +590,20 @@ static int run(int argc, char *argv[]) { if (!curl) { r = make_curl_easy_handle(&curl, write_chunk, &chunk_buffer, NULL); if (r < 0) - goto finish; + return r; } r = process_remote(rr, PROCESS_UNTIL_HAVE_REQUEST); - if (r == -EPIPE) { - r = 0; - goto finish; - } + if (r == -EPIPE) + return 0; if (r < 0) - goto finish; + return r; r = ca_remote_next_request(rr, &id); if (r == -ENODATA) continue; - if (r < 0) { - log_error_errno(r, "Failed to determine next chunk to get: %m"); - goto finish; - } + if (r < 0) + return log_error_errno(r, "Failed to determine next chunk to get: %m"); current_store = current_store % n_stores; if (wstore_url) @@ -625,44 +614,31 @@ static int run(int argc, char *argv[]) { free(url_buffer); url_buffer = chunk_url(store_url, &id); - if (!url_buffer) { - r = log_oom(); - goto finish; - } + if (!url_buffer) + return log_oom(); r = configure_curl_easy_handle(curl, url_buffer); if (r < 0) - goto finish; + return r; log_debug("Acquiring %s...", url_buffer); - if (robust_curl_easy_perform(curl) != CURLE_OK) { - log_error("Failed to acquire %s", url_buffer); - r = -EIO; - goto finish; - } + if (robust_curl_easy_perform(curl) != CURLE_OK) + return log_error_errno(-EIO, "Failed to acquire %s", url_buffer); - if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) { - log_error("Failed to query response code"); - r = -EIO; - goto finish; - } + if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) + return log_error_errno(-EIO, "Failed to query response code"); r = process_remote(rr, PROCESS_UNTIL_CAN_PUT_CHUNK); - if (r == -EPIPE) { - r = 0; - goto finish; - } + if (r == -EPIPE) + return 0; if (r < 0) - goto finish; + return r; if (protocol_status_ok(arg_protocol, protocol_status)) { r = ca_remote_put_chunk(rr, &id, CA_CHUNK_COMPRESSED, realloc_buffer_data(&chunk_buffer), realloc_buffer_size(&chunk_buffer)); - if (r < 0) { - log_error_errno(r, "Failed to write chunk: %m"); - goto finish; - } - + if (r < 0) + return log_error_errno(r, "Failed to write chunk: %m"); } else { if (arg_verbose) log_error("%s server failure %ld while requesting %s", @@ -670,30 +646,22 @@ static int run(int argc, char *argv[]) { url_buffer); r = ca_remote_put_missing(rr, &id); - if (r < 0) { - log_error_errno(r, "Failed to write missing message: %m"); - goto finish; - } + if (r < 0) + return log_error_errno(r, "Failed to write missing message: %m"); } realloc_buffer_empty(&chunk_buffer); r = process_remote(rr, PROCESS_UNTIL_WRITTEN); - if (r == -EPIPE) { - r = 0; - goto finish; - } + if (r == -EPIPE) + return 0; if (r < 0) - goto finish; + return r; } flush: r = process_remote(rr, PROCESS_UNTIL_FINISHED); -finish: - if (curl) - curl_easy_cleanup(curl); - return r; } From 60577489f33539eea44eb6d753b82f4c565102b3 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 14:13:58 +0700 Subject: [PATCH 10/14] casync-http: Log curl error code in acquire_file() Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 10e49174..a9f802f5 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -440,6 +440,7 @@ static char *chunk_url(const char *store_url, const CaChunkID *id) { } static int acquire_file(CaRemote *rr, CURL *handle) { + CURLcode c; long protocol_status; const char *url; @@ -448,15 +449,13 @@ static int acquire_file(CaRemote *rr, CURL *handle) { log_debug("Acquiring %s...", url); - if (robust_curl_easy_perform(handle) != CURLE_OK) { - log_error("Failed to acquire %s", url); - return -EIO; - } + c = robust_curl_easy_perform(handle); + if (c != CURLE_OK) + return log_error_curle(c, "Failed to acquire %s", url); - if (curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) { - log_error("Failed to query response code"); - return -EIO; - } + c = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &protocol_status); + if (c != CURLE_OK) + return log_error_curle(c, "Failed to query response code"); if (!protocol_status_ok(arg_protocol, protocol_status)) { _cleanup_free_ char *m = NULL; From 77652a890075b93aeb3a8444d28e92b15feaafe5 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 25 Jun 2019 16:59:48 +0700 Subject: [PATCH 11/14] casync-http: Use log_error_errno to report casync error code Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index a9f802f5..2052d786 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -315,7 +315,7 @@ static size_t write_index(const void *buffer, size_t size, size_t nmemb, void *u r = ca_remote_put_index(rr, buffer, product); if (r < 0) { - log_error("Failed to put index: %m"); + log_error_errno(r, "Failed to put index: %m"); return 0; } @@ -359,7 +359,7 @@ static size_t write_archive(const void *buffer, size_t size, size_t nmemb, void r = ca_remote_put_archive(rr, buffer, product); if (r < 0) { - log_error("Failed to put archive: %m"); + log_error_errno(r, "Failed to put archive: %m"); return 0; } From e6bfd49e253fad4cad03358b355d9ed7ea198f70 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 02:36:37 +0700 Subject: [PATCH 12/14] casync-http: Move chunks download in a separate function for clarity The goal is to make the run() function more readable, and only outline the major steps, while the bulk of the work is left to other functions. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 162 ++++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 76 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 2052d786..4e3739c4 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -481,14 +481,92 @@ static int acquire_file(CaRemote *rr, CURL *handle) { return 1; } +static int acquire_chunks(CaRemote *rr, const char *store_url) { + _cleanup_free_ char *url_buffer = NULL; + _cleanup_(curl_easy_cleanupp) CURL *curl = NULL; + _cleanup_(realloc_buffer_free) ReallocBuffer chunk_buffer = {}; + int r; + + r = make_curl_easy_handle(&curl, write_chunk, &chunk_buffer, NULL); + if (r < 0) + return r; + + for (;;) { + CURLcode c; + CaChunkID id; + long protocol_status; + + if (quit) { + log_info("Got exit signal, quitting."); + return 0; + } + + r = process_remote(rr, PROCESS_UNTIL_HAVE_REQUEST); + if (r == -EPIPE) + return 0; + if (r < 0) + return r; + + r = ca_remote_next_request(rr, &id); + if (r == -ENODATA) + continue; + if (r < 0) + return log_error_errno(r, "Failed to determine next chunk to get: %m"); + + free(url_buffer); + url_buffer = chunk_url(store_url, &id); + if (!url_buffer) + return log_oom(); + + r = configure_curl_easy_handle(curl, url_buffer); + if (r < 0) + return r; + + log_debug("Acquiring %s...", url_buffer); + + c = robust_curl_easy_perform(curl); + if (c != CURLE_OK) + return log_error_curle(c, "Failed to acquire %s", url_buffer); + + c = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &protocol_status); + if (c != CURLE_OK) + return log_error_curle(c, "Failed to query response code"); + + r = process_remote(rr, PROCESS_UNTIL_CAN_PUT_CHUNK); + if (r == -EPIPE) + return 0; + if (r < 0) + return r; + + if (protocol_status_ok(arg_protocol, protocol_status)) { + r = ca_remote_put_chunk(rr, &id, CA_CHUNK_COMPRESSED, realloc_buffer_data(&chunk_buffer), realloc_buffer_size(&chunk_buffer)); + if (r < 0) + return log_error_errno(r, "Failed to write chunk: %m"); + } else { + if (arg_verbose) + log_error("%s server failure %ld while requesting %s", + protocol_str(arg_protocol), protocol_status, + url_buffer); + + r = ca_remote_put_missing(rr, &id); + if (r < 0) + return log_error_errno(r, "Failed to write missing message: %m"); + } + + realloc_buffer_empty(&chunk_buffer); + + r = process_remote(rr, PROCESS_UNTIL_WRITTEN); + if (r == -EPIPE) + return 0; + if (r < 0) + return r; + } +} + static int run(int argc, char *argv[]) { const char *base_url, *archive_url, *index_url, *wstore_url; size_t n_stores = 0, current_store = 0; - _cleanup_(curl_easy_cleanupp) CURL *curl = NULL; _cleanup_(ca_remote_unrefp) CaRemote *rr = NULL; - _cleanup_(realloc_buffer_free) ReallocBuffer chunk_buffer = {}; - _cleanup_free_ char *url_buffer = NULL; - long protocol_status; int r; if (argc < _CA_REMOTE_ARG_MAX) { @@ -574,86 +652,18 @@ static int run(int argc, char *argv[]) { return r; } - for (;;) { + if (n_stores > 0) { const char *store_url; - CaChunkID id; - - if (quit) { - log_info("Got exit signal, quitting."); - return 0; - } - - if (n_stores == 0) /* No stores? Then we did all we could do */ - break; - - if (!curl) { - r = make_curl_easy_handle(&curl, write_chunk, &chunk_buffer, NULL); - if (r < 0) - return r; - } - - r = process_remote(rr, PROCESS_UNTIL_HAVE_REQUEST); - if (r == -EPIPE) - return 0; - if (r < 0) - return r; - - r = ca_remote_next_request(rr, &id); - if (r == -ENODATA) - continue; - if (r < 0) - return log_error_errno(r, "Failed to determine next chunk to get: %m"); current_store = current_store % n_stores; if (wstore_url) - store_url = current_store == 0 ? wstore_url : argv[current_store + _CA_REMOTE_ARG_MAX - 1]; + store_url = current_store == 0 ? wstore_url : + argv[current_store + _CA_REMOTE_ARG_MAX - 1]; else store_url = argv[current_store + _CA_REMOTE_ARG_MAX]; /* current_store++; */ - free(url_buffer); - url_buffer = chunk_url(store_url, &id); - if (!url_buffer) - return log_oom(); - - r = configure_curl_easy_handle(curl, url_buffer); - if (r < 0) - return r; - - log_debug("Acquiring %s...", url_buffer); - - if (robust_curl_easy_perform(curl) != CURLE_OK) - return log_error_errno(-EIO, "Failed to acquire %s", url_buffer); - - if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &protocol_status) != CURLE_OK) - return log_error_errno(-EIO, "Failed to query response code"); - - r = process_remote(rr, PROCESS_UNTIL_CAN_PUT_CHUNK); - if (r == -EPIPE) - return 0; - if (r < 0) - return r; - - if (protocol_status_ok(arg_protocol, protocol_status)) { - r = ca_remote_put_chunk(rr, &id, CA_CHUNK_COMPRESSED, realloc_buffer_data(&chunk_buffer), realloc_buffer_size(&chunk_buffer)); - if (r < 0) - return log_error_errno(r, "Failed to write chunk: %m"); - } else { - if (arg_verbose) - log_error("%s server failure %ld while requesting %s", - protocol_str(arg_protocol), protocol_status, - url_buffer); - - r = ca_remote_put_missing(rr, &id); - if (r < 0) - return log_error_errno(r, "Failed to write missing message: %m"); - } - - realloc_buffer_empty(&chunk_buffer); - - r = process_remote(rr, PROCESS_UNTIL_WRITTEN); - if (r == -EPIPE) - return 0; + r = acquire_chunks(rr, store_url); if (r < 0) return r; } From 7b5ab1716476ac9c037da691b1ce444620166d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20PORTAY?= Date: Thu, 13 Jun 2019 14:28:09 -0400 Subject: [PATCH 13/14] casync-http: use a constant for long option --rate-limit-bps This long option does not use a short option (see the optstring in call of getopt_long). Use a constant instead, as it is done in casync-tool.c. --- src/casync-http.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/casync-http.c b/src/casync-http.c index 4e3739c4..5f644f8c 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -680,10 +680,14 @@ static void help(void) { static int parse_argv(int argc, char *argv[]) { + enum { + ARG_RATE_LIMIT_BPS = 0x100, + }; + static const struct option options[] = { { "help", no_argument, NULL, 'h' }, { "verbose", no_argument, NULL, 'v' }, - { "rate-limit-bps", required_argument, NULL, 'l' }, + { "rate-limit-bps", required_argument, NULL, ARG_RATE_LIMIT_BPS }, {} }; @@ -720,7 +724,7 @@ static int parse_argv(int argc, char *argv[]) { arg_verbose = true; break; - case 'l': + case ARG_RATE_LIMIT_BPS: arg_rate_limit_bps = strtoll(optarg, NULL, 10); break; From bc85e2651fb537a45bad2b6d3d8f46e8909c7650 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Tue, 18 Jun 2019 03:08:18 +0700 Subject: [PATCH 14/14] casync-http: Add cmdline option to trust ssl peers This can be useful for testing, if ever we do HTTP2/SSL with a local, untrusted server. Signed-off-by: Arnaud Rebillout --- src/casync-http.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/casync-http.c b/src/casync-http.c index 5f644f8c..e1e22003 100644 --- a/src/casync-http.c +++ b/src/casync-http.c @@ -15,6 +15,7 @@ static volatile sig_atomic_t quit = false; static bool arg_verbose = false; static curl_off_t arg_rate_limit_bps = 0; +static bool arg_ssl_trust_peer = false; typedef enum Protocol { PROTOCOL_HTTP, @@ -162,6 +163,9 @@ static int make_curl_easy_handle(CURL **ret, if (private) CURL_SETOPT_EASY(h, CURLOPT_PRIVATE, private); + if (arg_ssl_trust_peer) + CURL_SETOPT_EASY(h, CURLOPT_SSL_VERIFYPEER, false); + /* CURL_SETOPT_EASY(h, CURLOPT_VERBOSE, 1L); */ *ret = TAKE_PTR(h); @@ -682,12 +686,14 @@ static int parse_argv(int argc, char *argv[]) { enum { ARG_RATE_LIMIT_BPS = 0x100, + ARG_SSL_TRUST_PEER, }; static const struct option options[] = { { "help", no_argument, NULL, 'h' }, { "verbose", no_argument, NULL, 'v' }, { "rate-limit-bps", required_argument, NULL, ARG_RATE_LIMIT_BPS }, + { "ssl-trust-peer", no_argument, NULL, ARG_SSL_TRUST_PEER }, {} }; @@ -728,6 +734,10 @@ static int parse_argv(int argc, char *argv[]) { arg_rate_limit_bps = strtoll(optarg, NULL, 10); break; + case ARG_SSL_TRUST_PEER: + arg_ssl_trust_peer = true; + break; + case '?': return -EINVAL;