From 48b3ffa406cdc4531ff9b32313cd99c45d5068ad Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 6 Feb 2021 22:15:08 +0100 Subject: [PATCH 01/10] lib/fetch: sync with freebsd --- lib/fetch/common.c | 664 +++++++++---------- lib/fetch/common.h | 65 +- lib/fetch/fetch.c | 293 +++------ lib/fetch/fetch.h | 58 +- lib/fetch/file.c | 54 +- lib/fetch/ftp.c | 515 +++++++-------- lib/fetch/http.c | 1516 ++++++++++++++++++++++++++++++-------------- 7 files changed, 1748 insertions(+), 1417 deletions(-) diff --git a/lib/fetch/common.c b/lib/fetch/common.c index 9a5ef7481..1ee21b175 100644 --- a/lib/fetch/common.c +++ b/lib/fetch/common.c @@ -1,7 +1,7 @@ -/* $FreeBSD: rev 288217 $ */ -/* $NetBSD: common.c,v 1.29 2014/01/08 20:25:34 joerg Exp $ */ /*- - * Copyright (c) 1998-2014 Dag-Erling Smorgrav + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1998-2016 Dag-Erling Smørgrav * Copyright (c) 2008, 2010 Joerg Sonnenberger * Copyright (c) 2013 Michael Gmelin * Copyright (c) 2019 Duncan Overbruck @@ -31,9 +31,10 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#define _DEFAULT_SOURCE #include "compat.h" -#include +#include #include #include #include @@ -43,23 +44,16 @@ #include #include -#if defined(HAVE_INTTYPES_H) || defined(NETBSD) -#include -#endif +#include #include +#include #include #include #include #include #include -#include #include -#include -#include - -#ifndef MSG_NOSIGNAL -#include -#endif +#include #ifdef WITH_SSL #include @@ -70,8 +64,8 @@ #include "fetch.h" #include "common.h" -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wformat-nonliteral" +#ifndef INFTIM +#define INFTIM (-1) #endif /*** Local data **************************************************************/ @@ -89,6 +83,10 @@ static struct fetcherr netdb_errlist[] = { { -1, FETCH_UNKNOWN, "Unknown resolver error" } }; +/* End-of-Line */ +static const char ENDL[2] = "\r\n"; + + /*** Error-reporting functions ***********************************************/ /* @@ -170,7 +168,7 @@ fetch_syserr(void) case EHOSTDOWN: fetchLastErrCode = FETCH_DOWN; break; -default: + default: fetchLastErrCode = FETCH_UNKNOWN; } snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno)); @@ -240,8 +238,6 @@ fetch_reopen(int sd) /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); - conn->ftp_home = NULL; - conn->cache_url = NULL; conn->next_buf = NULL; conn->next_len = 0; conn->sd = sd; @@ -589,10 +585,11 @@ happy_eyeballs_connect(struct addrinfo *res0, int verbose) } if (verbose) { - char hbuf[1025]; - if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, sizeof(hbuf), NULL, - 0, NI_NUMERICHOST) == 0) - fetch_info("connecting to %s", hbuf); + char hbuf[NI_MAXHOST]; + char pbuf[NI_MAXSERV]; + if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, sizeof(hbuf), pbuf, + sizeof(pbuf), NI_NUMERICHOST|NI_NUMERICSERV) == 0) + fetch_info("connecting to %s:%s", hbuf, pbuf); } if (connect(sd, res->ai_addr, res->ai_addrlen) == -1) { @@ -693,7 +690,7 @@ happy_eyeballs_connect(struct addrinfo *res0, int verbose) * Establish a TCP connection to the specified port on the specified host. */ conn_t * -fetch_connect(struct url *url, int af, int verbose) +fetch_connect(struct url *u, int af, int verbose) { conn_t *conn; char pbuf[10]; @@ -718,7 +715,7 @@ fetch_connect(struct url *url, int af, int verbose) socks_url->port = fetch_default_port(socks_url->scheme); connurl = socks_url; } else { - connurl = url; + connurl = u; } if (verbose) @@ -736,24 +733,18 @@ fetch_connect(struct url *url, int af, int verbose) return (NULL); } - if (verbose) - fetch_info("connecting to %s:%d", connurl->host, connurl->port); - sd = happy_eyeballs_connect(res0, verbose); freeaddrinfo(res0); if (sd == -1) { fetchFreeURL(socks_url); return (NULL); } - if ((conn = fetch_reopen(sd)) == NULL) { - fetchFreeURL(socks_url); - fetch_syserr(); - close(sd); - return NULL; - } + + if ((conn = fetch_reopen(sd)) == NULL) + goto syserr; if (socks_url) { if (strcasecmp(socks_url->scheme, SCHEME_SOCKS5) == 0) { - if (fetch_socks5(conn, url, socks_url, verbose) != 0) { + if (fetch_socks5(conn, u, socks_url, verbose) != 0) { fetchFreeURL(socks_url); fetch_syserr(); close(sd); @@ -763,9 +754,23 @@ fetch_connect(struct url *url, int af, int verbose) } fetchFreeURL(socks_url); } - conn->cache_url = fetchCopyURL(url); - conn->cache_af = af; + strlcpy(conn->scheme, u->scheme, sizeof(conn->scheme)); + strlcpy(conn->host, u->host, sizeof(conn->host)); + strlcpy(conn->user, u->user, sizeof(conn->user)); + strlcpy(conn->pwd, u->pwd, sizeof(conn->pwd)); + conn->port = u->port; + conn->af = af; + return (conn); +syserr: + fetch_syserr(); + goto fail; +fail: + if (socks_url) + fetchFreeURL(socks_url); + if (sd >= 0) + close(sd); + return (NULL); } static pthread_mutex_t cache_mtx = PTHREAD_MUTEX_INITIALIZER; @@ -800,10 +805,12 @@ fetchConnectionCacheClose(void) { conn_t *conn; + pthread_mutex_lock(&cache_mtx); while ((conn = connection_cache) != NULL) { - connection_cache = conn->next_cached; - (*conn->cache_close)(conn); + connection_cache = conn->next; + (*conn->close)(conn); } + pthread_mutex_unlock(&cache_mtx); } /* @@ -816,18 +823,18 @@ fetch_cache_get(const struct url *url, int af) conn_t *conn, *last_conn = NULL; pthread_mutex_lock(&cache_mtx); - for (conn = connection_cache; conn; conn = conn->next_cached) { - if (conn->cache_url->port == url->port && - strcmp(conn->cache_url->scheme, url->scheme) == 0 && - strcmp(conn->cache_url->host, url->host) == 0 && - strcmp(conn->cache_url->user, url->user) == 0 && - strcmp(conn->cache_url->pwd, url->pwd) == 0 && - (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC || - conn->cache_af == af)) { + for (conn = connection_cache; conn; conn = conn->next) { + if (conn->port == url->port && + strcmp(conn->scheme, url->scheme) == 0 && + strcmp(conn->host, url->host) == 0 && + strcmp(conn->user, url->user) == 0 && + strcmp(conn->pwd, url->pwd) == 0 && + (conn->af == AF_UNSPEC || af == AF_UNSPEC || + conn->af == af)) { if (last_conn != NULL) - last_conn->next_cached = conn->next_cached; + last_conn->next = conn->next; else - connection_cache = conn->next_cached; + connection_cache = conn->next; pthread_mutex_unlock(&cache_mtx); return conn; @@ -849,7 +856,7 @@ fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) conn_t *iter, *last; int global_count, host_count; - if (conn->cache_url == NULL || cache_global_limit == 0) { + if (cache_global_limit == 0) { (*closecb)(conn); return; } @@ -858,28 +865,27 @@ fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) global_count = host_count = 0; last = NULL; for (iter = connection_cache; iter; - last = iter, iter = iter->next_cached) { + last = iter, iter = iter->next) { ++global_count; - if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0) + if (strcmp(conn->host, iter->host) == 0) ++host_count; if (global_count < cache_global_limit && host_count < cache_per_host_limit) continue; --global_count; if (last != NULL) - last->next_cached = iter->next_cached; + last->next = iter->next; else - connection_cache = iter->next_cached; - (*iter->cache_close)(iter); + connection_cache = iter->next; + (*iter->close)(iter); } - conn->cache_close = closecb; - conn->next_cached = connection_cache; + conn->close = closecb; + conn->next = connection_cache; connection_cache = conn; pthread_mutex_unlock(&cache_mtx); } - #ifdef WITH_SSL #ifndef HAVE_STRNSTR @@ -905,7 +911,7 @@ strnstr(const char *s, const char *find, size_t slen) } while (strncmp(s, find, len) != 0); s--; } - return ((char *)__UNCONST(s)); + return __DECONST(char *, s); } #endif @@ -995,7 +1001,7 @@ fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len) * is usually part of subjectAltName or CN of a certificate presented to * the client. This includes wildcard matching. The algorithm is based on * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280. - */ + */ static int fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, size_t mlen) @@ -1023,7 +1029,7 @@ fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4) return (0); mdot1idx = mdot1 - m; - mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1); + mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1); if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2) return (0); /* hostname must contain a dot and not be the 1st char */ @@ -1056,7 +1062,7 @@ fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, if (!fetch_ssl_hname_equal(hdot - delta, delta, mdot1 - delta, delta)) return (0); - /* all tests succeded, it's a match */ + /* all tests succeeded, it's a match */ return (1); } @@ -1074,7 +1080,6 @@ fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len) host = malloc(len + 1); if (!host) return NULL; - memcpy(host, hostname, len); host[len] = '\0'; memset(&hints, 0, sizeof(hints)); @@ -1083,10 +1088,8 @@ fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len) hints.ai_protocol = 0; hints.ai_flags = AI_NUMERICHOST; /* port is not relevant for this purpose */ - if (getaddrinfo(host, "443", &hints, &res) != 0) { - free(host); - return NULL; - } + if (getaddrinfo(host, "443", &hints, &res) != 0) + res = NULL; free(host); return res; } @@ -1102,11 +1105,11 @@ fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost, if (lhost->ai_family == AF_INET && rhostlen == 4) { left = (void *)&((struct sockaddr_in*)(void *) - lhost->ai_addr)->sin_addr.s_addr; + lhost->ai_addr)->sin_addr.s_addr; #ifdef INET6 } else if (lhost->ai_family == AF_INET6 && rhostlen == 16) { left = (void *)&((struct sockaddr_in6 *)(void *) - lhost->ai_addr)->sin6_addr; + lhost->ai_addr)->sin6_addr; #endif } else return (0); @@ -1132,15 +1135,16 @@ fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r, if (laddr->ai_family == raddr->ai_family) { if (laddr->ai_family == AF_INET) { rip = (char *)&((struct sockaddr_in *)(void *) - raddr->ai_addr)->sin_addr.s_addr; + raddr->ai_addr)->sin_addr.s_addr; ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4); #ifdef INET6 } else if (laddr->ai_family == AF_INET6) { rip = (char *)&((struct sockaddr_in6 *)(void *) - raddr->ai_addr)->sin6_addr; + raddr->ai_addr)->sin6_addr; ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16); #endif } + } freeaddrinfo(raddr); return (ret); @@ -1159,8 +1163,24 @@ fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames, const char *ns; for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) { +#if OPENSSL_VERSION_NUMBER < 0x10000000L + /* + * This is a workaround, since the following line causes + * alignment issues in clang: + * name = sk_GENERAL_NAME_value(altnames, i); + * OpenSSL explicitly warns not to use those macros + * directly, but there isn't much choice (and there + * shouldn't be any ill side effects) + */ + name = (GENERAL_NAME *)SKM_sk_value(void, altnames, i); +#else name = sk_GENERAL_NAME_value(altnames, i); +#endif +#if OPENSSL_VERSION_NUMBER < 0x10100000L + ns = (const char *)ASN1_STRING_data(name->d.ia5); +#else ns = (const char *)ASN1_STRING_get0_data(name->d.ia5); +#endif nslen = (size_t)ASN1_STRING_length(name->d.ia5); if (name->type == GEN_DNS && ip == NULL && @@ -1191,7 +1211,7 @@ fetch_ssl_verify_cn(X509_NAME *subject, const char *host, cn = NULL; /* get most specific CN (last entry in list) and compare */ while ((lastpos = X509_NAME_get_index_by_NID(subject, - NID_commonName, lastpos)) != -1) + NID_commonName, lastpos)) != -1) loc = lastpos; if (loc > -1) { @@ -1225,7 +1245,7 @@ fetch_ssl_verify_hname(X509 *cert, const char *host) ret = 0; ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); altnames = X509_get_ext_d2i(cert, NID_subject_alt_name, - NULL, NULL); + NULL, NULL); if (altnames != NULL) { ret = fetch_ssl_verify_altname(altnames, host, ip); @@ -1277,8 +1297,7 @@ fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose) if (getenv("SSL_NO_VERIFY_PEER") == NULL) { ca_cert_file = getenv("SSL_CA_CERT_FILE"); - ca_cert_path = getenv("SSL_CA_CERT_PATH") != NULL ? - getenv("SSL_CA_CERT_PATH") : X509_get_default_cert_dir(); + ca_cert_path = getenv("SSL_CA_CERT_PATH"); if (verbose) { fetch_info("Peer verification enabled"); if (ca_cert_file != NULL) @@ -1287,15 +1306,20 @@ fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose) if (ca_cert_path != NULL) fetch_info("Using CA cert path: %s", ca_cert_path); + if (ca_cert_file == NULL && ca_cert_path == NULL) + fetch_info("Using OpenSSL default " + "CA cert file and path"); } SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, fetch_ssl_cb_verify_crt); - SSL_CTX_load_verify_locations(ctx, ca_cert_file, - ca_cert_path); + if (ca_cert_file != NULL || ca_cert_path != NULL) + SSL_CTX_load_verify_locations(ctx, ca_cert_file, + ca_cert_path); + else + SSL_CTX_set_default_verify_paths(ctx); if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) { if (verbose) fetch_info("Using CRL file: %s", crl_file); - crl_store = SSL_CTX_get_cert_store(ctx); crl_lookup = X509_STORE_add_lookup(crl_store, X509_LOOKUP_file()); @@ -1385,22 +1409,21 @@ ssl_init(void) } #endif - /* * Enable SSL on a connection. */ int fetch_ssl(conn_t *conn, const struct url *URL, int verbose) { - #ifdef WITH_SSL - int ret; + int ret, ssl_err; X509_NAME *name; char *str; (void)pthread_once(&ssl_init_once, ssl_init); - conn->ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + conn->ssl_meth = SSLv23_client_method(); + conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth); if (conn->ssl_ctx == NULL) { fprintf(stderr, "failed to create SSL context\n"); ERR_print_errors_fp(stderr); @@ -1425,18 +1448,22 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) return (-1); } #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) - if (!SSL_set_tlsext_host_name(conn->ssl, (char *)(uintptr_t)URL->host)) { + if (!SSL_set_tlsext_host_name(conn->ssl, + __DECONST(struct url *, URL)->host)) { fprintf(stderr, "TLS server name indication extension failed for host %s\n", URL->host); return (-1); } #endif - if ((ret = SSL_connect(conn->ssl)) <= 0){ - fprintf(stderr, "SSL_connect returned %d\n", SSL_get_error(conn->ssl, ret)); - return (-1); + while ((ret = SSL_connect(conn->ssl)) == -1) { + ssl_err = SSL_get_error(conn->ssl, ret); + if (ssl_err != SSL_ERROR_WANT_READ && + ssl_err != SSL_ERROR_WANT_WRITE) { + ERR_print_errors_fp(stderr); + return (-1); + } } - conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); if (conn->ssl_cert == NULL) { @@ -1444,13 +1471,13 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) return (-1); } - if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) { + if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) { if (verbose) fetch_info("Verify hostname"); if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) { fprintf(stderr, - "SSL certificate subject doesn't match host %s\n", - URL->host); + "SSL certificate subject doesn't match host %s\n", + URL->host); return (-1); } } @@ -1458,7 +1485,6 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) if (verbose) { fetch_info("%s connection established using %s", SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl)); - conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); name = X509_get_subject_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); fetch_info("Certificate subject: %s", str); @@ -1473,11 +1499,52 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) #else (void)conn; (void)verbose; + (void)URL; fprintf(stderr, "SSL support disabled\n"); return (-1); #endif } +#define FETCH_READ_WAIT -2 +#define FETCH_READ_ERROR -1 +#define FETCH_READ_DONE 0 + +#ifdef WITH_SSL +static ssize_t +fetch_ssl_read(SSL *ssl, char *buf, size_t len) +{ + ssize_t rlen; + int ssl_err; + + rlen = SSL_read(ssl, buf, len); + if (rlen < 0) { + ssl_err = SSL_get_error(ssl, rlen); + if (ssl_err == SSL_ERROR_WANT_READ || + ssl_err == SSL_ERROR_WANT_WRITE) { + return (FETCH_READ_WAIT); + } else { + ERR_print_errors_fp(stderr); + return (FETCH_READ_ERROR); + } + } + return (rlen); +} +#endif + +static ssize_t +fetch_socket_read(int sd, char *buf, size_t len) +{ + ssize_t rlen; + + rlen = read(sd, buf, len); + if (rlen < 0) { + if (errno == EAGAIN || (errno == EINTR && fetchRestartCalls)) + return (FETCH_READ_WAIT); + else + return (FETCH_READ_ERROR); + } + return (rlen); +} /* * Read a character from a connection w/ timeout @@ -1485,10 +1552,10 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) ssize_t fetch_read(conn_t *conn, char *buf, size_t len) { - struct timeval now, timeout, waittv; - fd_set readfds; + struct timeval now, timeout, delta; + struct pollfd pfd; ssize_t rlen; - int r; + int deltams; if (!buf) return -1; @@ -1504,51 +1571,68 @@ fetch_read(conn_t *conn, char *buf, size_t len) return len; } - if (fetchTimeout) { - FD_ZERO(&readfds); + if (fetchTimeout > 0) { gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } + deltams = INFTIM; + memset(&pfd, 0, sizeof pfd); + pfd.fd = conn->sd; + pfd.events = POLLIN | POLLERR; + for (;;) { - while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) { - FD_SET(conn->sd, &readfds); - gettimeofday(&now, NULL); - waittv.tv_sec = timeout.tv_sec - now.tv_sec; - waittv.tv_usec = timeout.tv_usec - now.tv_usec; - if (waittv.tv_usec < 0) { - waittv.tv_usec += 1000000; - waittv.tv_sec--; - } - if (waittv.tv_sec < 0) { - errno = ETIMEDOUT; - fetch_syserr(); - return (-1); - } - errno = 0; + /* + * The socket is non-blocking. Instead of the canonical + * poll() -> read(), we do the following: + * + * 1) call read() or SSL_read(). + * 2) if we received some data, return it. + * 3) if an error occurred, return -1. + * 4) if read() or SSL_read() signaled EOF, return. + * 5) if we did not receive any data but we're not at EOF, + * call poll(). + * + * In the SSL case, this is necessary because if we + * receive a close notification, we have to call + * SSL_read() one additional time after we've read + * everything we received. + * + * In the non-SSL case, it may improve performance (very + * slightly) when reading small amounts of data. + */ #ifdef WITH_SSL - if (conn->ssl && SSL_pending(conn->ssl)) - break; + if (conn->ssl != NULL) + rlen = fetch_ssl_read(conn->ssl, buf, len); + else #endif - r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv); - if (r == -1) { - if (errno == EINTR && fetchRestartCalls) - continue; + rlen = fetch_socket_read(conn->sd, buf, len); + if (rlen >= 0) { + break; + } else if (rlen == FETCH_READ_ERROR) { + fetch_syserr(); + return (-1); + } + // assert(rlen == FETCH_READ_WAIT); + if (fetchTimeout > 0) { + gettimeofday(&now, NULL); + if (!timercmp(&timeout, &now, >)) { + errno = ETIMEDOUT; fetch_syserr(); return (-1); } + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000;; } -#ifdef WITH_SSL - if (conn->ssl != NULL) - rlen = SSL_read(conn->ssl, buf, len); - else -#endif - rlen = read(conn->sd, buf, len); - if (rlen >= 0) - break; - - if (errno != EINTR || !fetchRestartCalls) + errno = 0; + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { + if (errno == EINTR && fetchRestartCalls) + continue; + fetch_syserr(); return (-1); + } } return (rlen); } @@ -1616,57 +1700,62 @@ fetch_getln(conn_t *conn) conn->buf[conn->buflen] = '\0'; conn->next_len = 0; } + DEBUGF("<<< %s\n", conn->buf); return (0); } + +/* + * Write to a connection w/ timeout + */ +ssize_t +fetch_write(conn_t *conn, const char *buf, size_t len) +{ + struct iovec iov; + + iov.iov_base = __DECONST(char *, buf); + iov.iov_len = len; + return fetch_writev(conn, &iov, 1); +} + /* * Write a vector to a connection w/ timeout * Note: can modify the iovec. */ ssize_t -fetch_write(conn_t *conn, const void *buf, size_t len) +fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) { - struct timeval now, timeout, waittv; - fd_set writefds; + struct timeval now, timeout, delta; + struct pollfd pfd; ssize_t wlen, total; - int r; -#ifndef MSG_NOSIGNAL - static int killed_sigpipe; -#endif - -#ifndef MSG_NOSIGNAL - if (!killed_sigpipe) { - signal(SIGPIPE, SIG_IGN); - killed_sigpipe = 1; - } -#endif - + int deltams; + memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { - FD_ZERO(&writefds); + pfd.fd = conn->sd; + pfd.events = POLLOUT | POLLERR; gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } total = 0; - while (len) { - while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { - FD_SET(conn->sd, &writefds); + while (iovcnt > 0) { + while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); - waittv.tv_sec = timeout.tv_sec - now.tv_sec; - waittv.tv_usec = timeout.tv_usec - now.tv_usec; - if (waittv.tv_usec < 0) { - waittv.tv_usec += 1000000; - waittv.tv_sec--; - } - if (waittv.tv_sec < 0) { + if (!timercmp(&timeout, &now, >)) { errno = ETIMEDOUT; fetch_syserr(); return (-1); } + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000; errno = 0; - r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv); - if (r == -1) { + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { + /* POSIX compliance */ + if (errno == EAGAIN) + continue; if (errno == EINTR && fetchRestartCalls) continue; return (-1); @@ -1675,16 +1764,14 @@ fetch_write(conn_t *conn, const void *buf, size_t len) errno = 0; #ifdef WITH_SSL if (conn->ssl != NULL) - wlen = SSL_write(conn->ssl, buf, len); + wlen = SSL_write(conn->ssl, + iov->iov_base, iov->iov_len); else #endif -#ifndef MSG_NOSIGNAL - wlen = send(conn->sd, buf, len, 0); -#else - wlen = send(conn->sd, buf, len, MSG_NOSIGNAL); -#endif + wlen = writev(conn->sd, iov, iovcnt); if (wlen == 0) { /* we consider a short write a failure */ + /* XXX perhaps we shouldn't in the SSL case */ errno = EPIPE; fetch_syserr(); return (-1); @@ -1695,13 +1782,44 @@ fetch_write(conn_t *conn, const void *buf, size_t len) return (-1); } total += wlen; - buf = (const char *)buf + wlen; - len -= wlen; + while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) { + wlen -= iov->iov_len; + iov++; + iovcnt--; + } + if (iovcnt > 0) { + iov->iov_len -= wlen; + iov->iov_base = __DECONST(char *, iov->iov_base) + wlen; + } } return (total); } +/* + * Write a line of text to a connection w/ timeout + */ +int +fetch_putln(conn_t *conn, const char *str, size_t len) +{ + struct iovec iov[2]; + int ret; + + DEBUGF(">>> %s\n", str); + iov[0].iov_base = __DECONST(char *, str); + iov[0].iov_len = len; + iov[1].iov_base = __DECONST(char *, ENDL); + iov[1].iov_len = sizeof(ENDL); + if (len == 0) + ret = fetch_writev(conn, &iov[1], 1); + else + ret = fetch_writev(conn, iov, 2); + if (ret == -1) + return (-1); + return (0); +} + + /* * Close connection */ @@ -1710,6 +1828,8 @@ fetch_close(conn_t *conn) { int ret; + if (--conn->ref > 0) + return (0); #ifdef WITH_SSL if (conn->ssl) { SSL_shutdown(conn->ssl); @@ -1727,155 +1847,12 @@ fetch_close(conn_t *conn) } #endif ret = close(conn->sd); - if (conn->cache_url) - fetchFreeURL(conn->cache_url); - free(conn->ftp_home); free(conn->buf); free(conn); return (ret); } -/*** Directory-related utility functions *************************************/ - -int -fetch_add_entry(struct url_list *ue, struct url *base, const char *name, - int pre_quoted) -{ - struct url *tmp; - char *tmp_name; - size_t base_doc_len, name_len, i; - unsigned char c; - - if (strchr(name, '/') != NULL || - strcmp(name, "..") == 0 || - strcmp(name, ".") == 0) - return 0; - - if (strcmp(base->doc, "/") == 0) - base_doc_len = 0; - else - base_doc_len = strlen(base->doc); - - name_len = 1; - for (i = 0; name[i] != '\0'; ++i) { - if ((!pre_quoted && name[i] == '%') || - !fetch_urlpath_safe(name[i])) - name_len += 3; - else - ++name_len; - } - - tmp_name = malloc( base_doc_len + name_len + 1); - if (tmp_name == NULL) { - errno = ENOMEM; - fetch_syserr(); - return (-1); - } - - if (ue->length + 1 >= ue->alloc_size) { - tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp)); - if (tmp == NULL) { - free(tmp_name); - errno = ENOMEM; - fetch_syserr(); - return (-1); - } - ue->alloc_size = ue->alloc_size * 2 + 1; - ue->urls = tmp; - } - - tmp = ue->urls + ue->length; - strcpy(tmp->scheme, base->scheme); - strcpy(tmp->user, base->user); - strcpy(tmp->pwd, base->pwd); - strcpy(tmp->host, base->host); - tmp->port = base->port; - tmp->doc = tmp_name; - memcpy(tmp->doc, base->doc, base_doc_len); - tmp->doc[base_doc_len] = '/'; - - for (i = base_doc_len + 1; *name != '\0'; ++name) { - if ((!pre_quoted && *name == '%') || - !fetch_urlpath_safe(*name)) { - tmp->doc[i++] = '%'; - c = (unsigned char)*name / 16; - if (c < 10) - tmp->doc[i++] = '0' + c; - else - tmp->doc[i++] = 'a' - 10 + c; - c = (unsigned char)*name % 16; - if (c < 10) - tmp->doc[i++] = '0' + c; - else - tmp->doc[i++] = 'a' - 10 + c; - } else { - tmp->doc[i++] = *name; - } - } - tmp->doc[i] = '\0'; - - tmp->offset = 0; - tmp->length = 0; - tmp->last_modified = -1; - - ++ue->length; - - return (0); -} - -void -fetchInitURLList(struct url_list *ue) -{ - ue->length = ue->alloc_size = 0; - ue->urls = NULL; -} - -int -fetchAppendURLList(struct url_list *dst, const struct url_list *src) -{ - size_t i, j, len; - - len = dst->length + src->length; - if (len > dst->alloc_size) { - struct url *tmp; - - tmp = realloc(dst->urls, len * sizeof(*tmp)); - if (tmp == NULL) { - errno = ENOMEM; - fetch_syserr(); - return (-1); - } - dst->alloc_size = len; - dst->urls = tmp; - } - - for (i = 0, j = dst->length; i < src->length; ++i, ++j) { - dst->urls[j] = src->urls[i]; - dst->urls[j].doc = strdup(src->urls[i].doc); - if (dst->urls[j].doc == NULL) { - while (i-- > 0) - free(dst->urls[j].doc); - fetch_syserr(); - return -1; - } - } - dst->length = len; - - return 0; -} - -void -fetchFreeURLList(struct url_list *ue) -{ - size_t i; - - for (i = 0; i < ue->length; ++i) - free(ue->urls[i].doc); - free(ue->urls); - ue->length = ue->alloc_size = 0; -} - /*** Authentication-related utility functions ********************************/ @@ -1889,27 +1866,23 @@ fetch_read_word(FILE *f) return (word); } -/* - * Get authentication data for a URL from .netrc - */ -int -fetch_netrc_auth(struct url *url) +static int +fetch_netrc_open(void) { + struct passwd *pwd; char fn[PATH_MAX]; - const char *word; - char *p; - FILE *f; + const char *p; + int fd, serrno; if ((p = getenv("NETRC")) != NULL) { + DEBUGF("NETRC=%s\n", p); if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) { fetch_info("$NETRC specifies a file name " "longer than PATH_MAX"); return (-1); } } else { - if ((p = getenv("HOME")) != NULL) { - struct passwd *pwd; - + if ((p = getenv("HOME")) == NULL) { if ((pwd = getpwuid(getuid())) == NULL || (p = pwd->pw_dir) == NULL) return (-1); @@ -1918,14 +1891,47 @@ fetch_netrc_auth(struct url *url) return (-1); } - if ((f = fopen(fn, "r")) == NULL) + if ((fd = open(fn, O_RDONLY)) < 0) { + serrno = errno; + DEBUGF("%s: %s\n", fn, strerror(serrno)); + errno = serrno; + } + return (fd); +} + +/* + * Get authentication data for a URL from .netrc + */ +int +fetch_netrc_auth(struct url *url) +{ + const char *word; + int serrno; + FILE *f; + + if (url->netrcfd < 0) + url->netrcfd = fetch_netrc_open(); + if (url->netrcfd < 0) return (-1); + if ((f = fdopen(url->netrcfd, "r")) == NULL) { + serrno = errno; + DEBUGF("fdopen(netrcfd): %s", strerror(errno)); + close(url->netrcfd); + url->netrcfd = -1; + errno = serrno; + return (-1); + } + rewind(f); + DEBUGF("searching netrc for %s\n", url->host); while ((word = fetch_read_word(f)) != NULL) { - if (strcmp(word, "default") == 0) + if (strcmp(word, "default") == 0) { + DEBUGF("using default netrc settings\n"); break; + } if (strcmp(word, "machine") == 0 && (word = fetch_read_word(f)) != NULL && strcasecmp(word, url->host) == 0) { + DEBUGF("using netrc settings for %s\n", word); break; } } @@ -1957,9 +1963,13 @@ fetch_netrc_auth(struct url *url) } } fclose(f); + url->netrcfd = -1; return (0); - ferr: +ferr: + serrno = errno; fclose(f); + url->netrcfd = -1; + errno = serrno; return (-1); } @@ -1968,7 +1978,7 @@ fetch_netrc_auth(struct url *url) * which the proxy should not be consulted; the contents is a comma-, * or space-separated list of domain names. A single asterisk will * override all proxy variables and no transactions will be proxied - * (for compatability with lynx and curl, see the discussion at + * (for compatibility with lynx and curl, see the discussion at * ). */ int @@ -1998,7 +2008,7 @@ fetch_no_proxy_match(const char *host) break; d_len = q - p; - if (d_len > 0 && h_len > d_len && + if (d_len > 0 && h_len >= d_len && strncasecmp(host + h_len - d_len, p, d_len) == 0) { /* domain name matches */ @@ -2015,22 +2025,24 @@ struct fetchIO { void *io_cookie; ssize_t (*io_read)(void *, void *, size_t); ssize_t (*io_write)(void *, const void *, size_t); - void (*io_close)(void *); + int (*io_close)(void *); }; -void +int fetchIO_close(fetchIO *f) { + int rv = 0; if (f->io_close != NULL) - (*f->io_close)(f->io_cookie); + rv = (*f->io_close)(f->io_cookie); free(f); + return rv; } fetchIO * fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t), ssize_t (*io_write)(void *, const void *, size_t), - void (*io_close)(void *)) + int (*io_close)(void *)) { fetchIO *f; diff --git a/lib/fetch/common.h b/lib/fetch/common.h index 647b184b2..3ddb94018 100644 --- a/lib/fetch/common.h +++ b/lib/fetch/common.h @@ -1,7 +1,7 @@ -/* $FreeBSD: rev 267133 $ */ -/* $NetBSD: common.h,v 1.23 2014/01/08 20:25:34 joerg Exp $ */ /*- - * Copyright (c) 1998-2014 Dag-Erling Smorgrav + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,6 +26,8 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD: head/lib/libfetch/common.h 334317 2018-05-29 10:28:20Z des $ */ #ifndef _COMMON_H_INCLUDED @@ -98,14 +100,17 @@ struct fetchconn { SSL *ssl; /* SSL handle */ SSL_CTX *ssl_ctx; /* SSL context */ X509 *ssl_cert; /* server certificate */ + const SSL_METHOD *ssl_meth; /* SSL method */ #endif - - char *ftp_home; - - struct url *cache_url; - int cache_af; - int (*cache_close)(conn_t *); - conn_t *next_cached; + int ref; /* reference count */ + char scheme[URL_SCHEMELEN+1]; + char user[URL_USERLEN+1]; + char pwd[URL_PWDLEN+1]; + char host[URL_HOSTLEN+1]; + int port; + int af; + int (*close)(conn_t *); + conn_t *next; }; /* Structure used for error message lists */ @@ -115,28 +120,33 @@ struct fetcherr { const char *string; }; +/* for fetch_writev */ +struct iovec; + void fetch_seterr(struct fetcherr *, int); void fetch_syserr(void); void fetch_info(const char *, ...) LIBFETCH_PRINTFLIKE(1, 2); int fetch_default_port(const char *); int fetch_default_proxy_port(const char *); int fetch_bind(int, int, const char *); -conn_t *fetch_cache_get(const struct url *, int); -void fetch_cache_put(conn_t *, int (*)(conn_t *)); int fetch_socks5(conn_t *, struct url *, struct url *, int); conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); +conn_t *fetch_ref(conn_t *); #ifdef WITH_SSL -int fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*); +int fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*); #endif int fetch_ssl(conn_t *, const struct url *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); -ssize_t fetch_write(conn_t *, const void *, size_t); +ssize_t fetch_write(conn_t *, const char *, size_t); +ssize_t fetch_writev(conn_t *, struct iovec *, int); +int fetch_putln(conn_t *, const char *, size_t); int fetch_close(conn_t *); -int fetch_add_entry(struct url_list *, struct url *, const char *, int); int fetch_netrc_auth(struct url *url); int fetch_no_proxy_match(const char *); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)); int fetch_urlpath_safe(char); #define ftp_seterr(n) fetch_seterr(ftp_errlist, n) @@ -145,7 +155,20 @@ int fetch_urlpath_safe(char); #define url_seterr(n) fetch_seterr(url_errlist, n) fetchIO *fetchIO_unopen(void *, ssize_t (*)(void *, void *, size_t), - ssize_t (*)(void *, const void *, size_t), void (*)(void *)); + ssize_t (*)(void *, const void *, size_t), int (*)(void *)); + +#ifndef NDEBUG +#define DEBUGF(...) \ + do { \ + if (fetchDebug) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) +#else +#define DEBUGF(...) \ + do { \ + /* nothing */ \ + } while (0) +#endif /* * I don't really like exporting http_request() and ftp_request(), @@ -158,17 +181,19 @@ fetchIO *fetchIO_unopen(void *, ssize_t (*)(void *, void *, size_t), */ fetchIO *http_request(struct url *, const char *, struct url_stat *, struct url *, const char *); -fetchIO *ftp_request(struct url *, const char *, const char *, +fetchIO *http_request_body(struct url *, const char *, + struct url_stat *, struct url *, const char *, + const char *, const char *); +fetchIO *ftp_request(struct url *, const char *, struct url_stat *, struct url *, const char *); - /* * Check whether a particular flag is set */ #define CHECK_FLAG(x) (flags && strchr(flags, (x))) -#ifndef __UNCONST -#define __UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) +#ifndef __DECONST +#define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) #endif #endif diff --git a/lib/fetch/fetch.c b/lib/fetch/fetch.c index fbf6c6fd7..195350d80 100644 --- a/lib/fetch/fetch.c +++ b/lib/fetch/fetch.c @@ -48,7 +48,7 @@ int fetchTimeout; int fetchConnTimeout = 300 * 1000; int fetchConnDelay = 250; volatile int fetchRestartCalls = 1; -int fetchDebug; +int fetchDebug = 1; /*** Local data **************************************************************/ @@ -148,26 +148,6 @@ fetchStat(struct url *URL, struct url_stat *us, const char *flags) return (-1); } -/* - * Select the appropriate protocol for the URL scheme, and return a - * list of files in the directory pointed to by the URL. - */ -int -fetchList(struct url_list *ue, struct url *URL, const char *pattern, - const char *flags) -{ - - if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) - return (fetchListFile(ue, URL, pattern, flags)); - else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) - return (fetchListFTP(ue, URL, pattern, flags)); - else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0) - return (fetchListHTTP(ue, URL, pattern, flags)); - else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0) - return (fetchListHTTP(ue, URL, pattern, flags)); - url_seterr(URL_BAD_SCHEME); - return -1; -} /* * Attempt to parse the given URL; if successful, call fetchXGet(). @@ -232,25 +212,6 @@ fetchStatURL(const char *URL, struct url_stat *us, const char *flags) return (s); } -/* - * Attempt to parse the given URL; if successful, call fetchList(). - */ -int -fetchListURL(struct url_list *ue, const char *URL, const char *pattern, - const char *flags) -{ - struct url *u; - int rv; - - if ((u = fetchParseURL(URL)) == NULL) - return -1; - - rv = fetchList(ue, u, pattern, flags); - - fetchFreeURL(u); - return rv; -} - /* * Make a URL */ @@ -275,6 +236,7 @@ fetchMakeURL(const char *scheme, const char *host, int port, const char *doc, fetch_syserr(); return (NULL); } + u->netrcfd = -1; if ((u->doc = strdup(doc ? doc : "/")) == NULL) { fetch_syserr(); @@ -299,6 +261,7 @@ fetchMakeURL(const char *scheme, const char *host, int port, const char *doc, static int fetch_hexval(char ch) { + if (ch >= '0' && ch <= '9') return (ch - '0'); else if (ch >= 'a' && ch <= 'f') @@ -337,66 +300,6 @@ fetch_pctdecode(char *dst, const char *src, size_t dlen) return (s); } -int -fetch_urlpath_safe(char x) -{ - if ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'Z') || - (x >= 'a' && x <= 'z')) - return 1; - - switch (x) { - case '$': - case '-': - case '_': - case '.': - case '+': - case '!': - case '*': - case '\'': - case '(': - case ')': - case ',': - /* The following are allowed in segment and path components: */ - case '?': - case ':': - case '@': - case '&': - case '=': - case '/': - case ';': - /* If something is already quoted... */ - case '%': - return 1; - default: - return 0; - } -} - -/* - * Copy an existing URL. - */ -struct url * -fetchCopyURL(const struct url *src) -{ - struct url *dst; - char *doc; - - /* allocate struct url */ - if ((dst = malloc(sizeof(*dst))) == NULL) { - fetch_syserr(); - return (NULL); - } - if ((doc = strdup(src->doc)) == NULL) { - fetch_syserr(); - free(dst); - return (NULL); - } - *dst = *src; - dst->doc = doc; - - return dst; -} - /* * Split an URL into components. URL syntax is: * [method:/][/[user[:pwd]@]host[:port]/][document] @@ -405,86 +308,46 @@ fetchCopyURL(const struct url *src) struct url * fetchParseURL(const char *URL) { + char *doc; const char *p, *q; struct url *u; - size_t i, count; - int pre_quoted; + int i, n; /* allocate struct url */ if ((u = calloc(1, sizeof(*u))) == NULL) { fetch_syserr(); return (NULL); } - - if (*URL == '/') { - pre_quoted = 0; - strcpy(u->scheme, SCHEME_FILE); - p = URL; - goto quote_doc; - } - if (strncmp(URL, "file:", 5) == 0) { - pre_quoted = 1; - strcpy(u->scheme, SCHEME_FILE); - URL += 5; - if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') { - url_seterr(URL_MALFORMED); - goto ouch; - } - URL += 2; + u->netrcfd = -1; + + /* scheme name */ + if ((p = strstr(URL, ":/"))) { + if (p - URL > URL_SCHEMELEN) + goto ouch; + for (i = 0; URL + i < p; i++) + u->scheme[i] = tolower((unsigned char)URL[i]); + URL = ++p; + /* + * Only one slash: no host, leave slash as part of document + * Two slashes: host follows, strip slashes + */ + if (URL[1] == '/') + URL = (p += 2); + } else { p = URL; - goto quote_doc; - } - if (strncmp(URL, "http:", 5) == 0 || - strncmp(URL, "https:", 6) == 0) { - pre_quoted = 1; - if (URL[4] == ':') { - strcpy(u->scheme, SCHEME_HTTP); - URL += 5; - } else { - strcpy(u->scheme, SCHEME_HTTPS); - URL += 6; - } - - if (URL[0] != '/' || URL[1] != '/') { - url_seterr(URL_MALFORMED); - goto ouch; - } - URL += 2; - goto find_user; - } - if (strncmp(URL, "ftp:", 4) == 0) { - pre_quoted = 1; - strcpy(u->scheme, SCHEME_FTP); - URL += 4; - if (URL[0] != '/' || URL[1] != '/') { - url_seterr(URL_MALFORMED); - goto ouch; - } - URL += 2; - goto find_user; - } - if (strncmp(URL, "socks5:", 7) == 0) { - pre_quoted = 1; - strcpy(u->scheme, SCHEME_SOCKS5); - URL += 7; - if (URL[0] != '/' || URL[1] != '/') { - url_seterr(URL_MALFORMED); - goto ouch; - } - URL += 2; - goto find_user; } + if (!*URL || *URL == '/' || *URL == '.' || + (u->scheme[0] == '\0' && + strchr(URL, '/') == NULL && strchr(URL, ':') == NULL)) + goto nohost; - url_seterr(URL_BAD_SCHEME); - goto ouch; - -find_user: p = strpbrk(URL, "/@"); - if (p != NULL && *p == '@') { + if (p && *p == '@') { /* username */ q = fetch_pctdecode(u->user, URL, URL_USERLEN); if (q == NULL) goto ouch; + /* password */ if (*q == ':') { q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN); @@ -497,66 +360,84 @@ fetchParseURL(const char *URL) } /* hostname */ -#ifdef INET6 - if (*p == '[' && (q = strchr(p + 1, ']')) != NULL && - (*++q == '\0' || *q == '/' || *q == ':')) { - if ((i = q - p - 2) > URL_HOSTLEN) - i = URL_HOSTLEN; - strncpy(u->host, ++p, i); - p = q; - } else -#endif - for (i = 0; *p && (*p != '/') && (*p != ':'); p++) - if (i < URL_HOSTLEN) - u->host[i++] = *p; + if (*p == '[') { + q = p + 1 + strspn(p + 1, ":0123456789ABCDEFabcdef"); + if (*q++ != ']') + goto ouch; + } else { + /* valid characters in a DNS name */ + q = p + strspn(p, "-." "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_" + "abcdefghijklmnopqrstuvwxyz"); + } + if ((*q != '\0' && *q != '/' && *q != ':') || q - p > URL_HOSTLEN) + goto ouch; + for (i = 0; p + i < q; i++) + u->host[i] = tolower((unsigned char)p[i]); + u->host[i] = '\0'; + p = q; /* port */ if (*p == ':') { - for (q = ++p; *q && (*q != '/'); q++) - if (isdigit((unsigned char)*q)) - u->port = u->port * 10 + (*q - '0'); - else { + for (n = 0, q = ++p; *q && (*q != '/'); q++) { + if (*q >= '0' && *q <= '9' && n < INT_MAX / 10) { + n = n * 10 + (*q - '0'); + } else { /* invalid port */ url_seterr(URL_BAD_PORT); goto ouch; } + } + /* pkg extension allow for ssh compat */ + /*if (n < 1 || n > IPPORT_MAX) */ +#ifndef IPPORT_MAX +#define IPPORT_MAX 65535 +#endif + if (n < 0 || n > IPPORT_MAX) + goto ouch; + u->port = n; p = q; } +nohost: /* document */ if (!*p) p = "/"; -quote_doc: - count = 1; - for (i = 0; p[i] != '\0'; ++i) { - if ((!pre_quoted && p[i] == '%') || - !fetch_urlpath_safe(p[i])) - count += 3; - else - ++count; - } + if (strcmp(u->scheme, SCHEME_HTTP) == 0 || + strcmp(u->scheme, SCHEME_HTTPS) == 0) { + const char hexnums[] = "0123456789abcdef"; - if ((u->doc = malloc(count)) == NULL) { + /* percent-escape whitespace. */ + if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) { + fetch_syserr(); + goto ouch; + } + u->doc = doc; + while (*p != '\0') { + if (!isspace((unsigned char)*p)) { + *doc++ = *p++; + } else { + *doc++ = '%'; + *doc++ = hexnums[((unsigned int)*p) >> 4]; + *doc++ = hexnums[((unsigned int)*p) & 0xf]; + p++; + } + } + *doc = '\0'; + } else if ((u->doc = strdup(p)) == NULL) { fetch_syserr(); goto ouch; } - for (i = 0; *p != '\0'; ++p) { - if ((!pre_quoted && *p == '%') || - !fetch_urlpath_safe(*p)) { - u->doc[i++] = '%'; - if ((unsigned char)*p < 160) - u->doc[i++] = '0' + ((unsigned char)*p) / 16; - else - u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16; - if ((unsigned char)*p % 16 < 10) - u->doc[i++] = '0' + ((unsigned char)*p) % 16; - else - u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16; - } else - u->doc[i++] = *p; - } - u->doc[i] = '\0'; + + DEBUGF("scheme: \"%s\"\n" + "user: \"%s\"\n" + "password: \"%s\"\n" + "host: \"%s\"\n" + "port: \"%d\"\n" + "document: \"%s\"\n", + u->scheme, u->user, u->pwd, + u->host, u->port, u->doc); return (u); diff --git a/lib/fetch/fetch.h b/lib/fetch/fetch.h index 9166aa508..6328c7cda 100644 --- a/lib/fetch/fetch.h +++ b/lib/fetch/fetch.h @@ -1,6 +1,8 @@ /* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */ /*- - * Copyright (c) 1998-2014 Dag-Erling Smorgrav + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,6 +27,8 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD: head/lib/libfetch/fetch.h 326219 2017-11-26 02:00:33Z pfg $ */ #ifndef _FETCH_H_INCLUDED @@ -44,15 +48,16 @@ typedef struct fetchIO fetchIO; struct url { - char scheme[URL_SCHEMELEN + 1]; - char user[URL_USERLEN + 1]; - char pwd[URL_PWDLEN + 1]; - char host[URL_HOSTLEN + 1]; + char scheme[URL_SCHEMELEN+1]; + char user[URL_USERLEN+1]; + char pwd[URL_PWDLEN+1]; + char host[URL_HOSTLEN+1]; int port; char *doc; off_t offset; size_t length; - time_t last_modified; + time_t ims_time; + int netrcfd; }; struct url_stat { @@ -61,12 +66,6 @@ struct url_stat { time_t mtime; }; -struct url_list { - size_t length; - size_t alloc_size; - struct url *urls; -}; - /* Recognized schemes */ #define SCHEME_FTP "ftp" #define SCHEME_HTTP "http" @@ -100,7 +99,7 @@ struct url_list { extern "C" { #endif -void fetchIO_close(fetchIO *); +int fetchIO_close(fetchIO *); ssize_t fetchIO_read(fetchIO *, void *, size_t); ssize_t fetchIO_write(fetchIO *, const void *, size_t); @@ -109,57 +108,48 @@ fetchIO *fetchXGetFile(struct url *, struct url_stat *, const char *); fetchIO *fetchGetFile(struct url *, const char *); fetchIO *fetchPutFile(struct url *, const char *); int fetchStatFile(struct url *, struct url_stat *, const char *); -int fetchListFile(struct url_list *, struct url *, const char *, - const char *); +struct url_ent *fetchListFile(struct url *, const char *); /* HTTP-specific functions */ fetchIO *fetchXGetHTTP(struct url *, struct url_stat *, const char *); fetchIO *fetchGetHTTP(struct url *, const char *); fetchIO *fetchPutHTTP(struct url *, const char *); int fetchStatHTTP(struct url *, struct url_stat *, const char *); -int fetchListHTTP(struct url_list *, struct url *, const char *, - const char *); +struct url_ent *fetchListHTTP(struct url *, const char *); +fetchIO *fetchReqHTTP(struct url *, const char *, const char *, + const char *, const char *); /* FTP-specific functions */ fetchIO *fetchXGetFTP(struct url *, struct url_stat *, const char *); fetchIO *fetchGetFTP(struct url *, const char *); fetchIO *fetchPutFTP(struct url *, const char *); int fetchStatFTP(struct url *, struct url_stat *, const char *); -int fetchListFTP(struct url_list *, struct url *, const char *, - const char *); +struct url_ent *fetchListFTP(struct url *, const char *); /* Generic functions */ fetchIO *fetchXGetURL(const char *, struct url_stat *, const char *); fetchIO *fetchGetURL(const char *, const char *); fetchIO *fetchPutURL(const char *, const char *); int fetchStatURL(const char *, struct url_stat *, const char *); -int fetchListURL(struct url_list *, const char *, const char *, - const char *); fetchIO *fetchXGet(struct url *, struct url_stat *, const char *); fetchIO *fetchGet(struct url *, const char *); fetchIO *fetchPut(struct url *, const char *); int fetchStat(struct url *, struct url_stat *, const char *); -int fetchList(struct url_list *, struct url *, const char *, - const char *); +struct url_ent *fetchList(struct url *, const char *); /* URL parsing */ struct url *fetchMakeURL(const char *, const char *, int, const char *, const char *, const char *); struct url *fetchParseURL(const char *); -struct url *fetchCopyURL(const struct url *); -char *fetchStringifyURL(const struct url *); +struct url *fetchDupURL(struct url *); +char *fetchStringifyURL(const struct url *); void fetchFreeURL(struct url *); - -/* URL listening */ -void fetchInitURLList(struct url_list *); -int fetchAppendURLList(struct url_list *, const struct url_list *); -void fetchFreeURLList(struct url_list *); -char *fetchUnquotePath(struct url *); -char *fetchUnquoteFilename(struct url *); +char *fetchUnquotePath(struct url *); +char *fetchUnquoteFilename(struct url *); /* Connection caching */ -void fetchConnectionCacheInit(int, int); -void fetchConnectionCacheClose(void); +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); /* Authentication */ typedef int (*auth_t)(struct url *); diff --git a/lib/fetch/file.c b/lib/fetch/file.c index 2089b86a7..0f263859e 100644 --- a/lib/fetch/file.c +++ b/lib/fetch/file.c @@ -1,7 +1,7 @@ -/* $NetBSD: file.c,v 1.15 2009/10/15 12:36:57 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav - * Copyright (c) 2008, 2009 Joerg Sonnenberger + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1998-2011 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,8 +26,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: file.c,v 1.18 2007/12/14 10:26:58 des Exp $ */ #include "compat.h" @@ -59,14 +57,14 @@ fetchFile_write(void *cookie, const void *buf, size_t len) return write(*(int *)cookie, buf, len); } -static void +static int fetchFile_close(void *cookie) { int fd = *(int *)cookie; free(cookie); - close(fd); + return close(fd); } fetchIO * @@ -99,8 +97,8 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags) return NULL; } - if (if_modified_since && u->last_modified > 0 && - u->last_modified >= us->mtime) { + if (if_modified_since && u->ims_time > 0 && + u->ims_time >= us->mtime) { close(fd); fetchLastErrCode = FETCH_UNCHANGED; snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); @@ -229,41 +227,3 @@ fetchStatFile(struct url *u, struct url_stat *us, const char *flags) return rv; } - -int -fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags) -{ - char *path; - struct dirent *de; - DIR *dir; - int ret; - - (void)flags; - - if ((path = fetchUnquotePath(u)) == NULL) { - fetch_syserr(); - return -1; - } - - dir = opendir(path); - free(path); - - if (dir == NULL) { - fetch_syserr(); - return -1; - } - - ret = 0; - - while ((de = readdir(dir)) != NULL) { - if (pattern && fnmatch(pattern, de->d_name, 0) != 0) - continue; - ret = fetch_add_entry(ue, u, de->d_name, 0); - if (ret) - break; - } - - closedir(dir); - - return ret; -} diff --git a/lib/fetch/ftp.c b/lib/fetch/ftp.c index 28296a009..2a4d0b380 100644 --- a/lib/fetch/ftp.c +++ b/lib/fetch/ftp.c @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.46 2014/06/11 13:12:12 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Smorgrav - * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger + * SPDX-License-Identifier: (BSD-3-Clause AND Beerware) + * + * Copyright (c) 1998-2011 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,8 +26,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: ftp.c,v 1.101 2008/01/23 20:57:59 des Exp $ */ /* @@ -42,7 +40,7 @@ * * Major Changelog: * - * Dag-Erling Smograv + * Dag-Erling Smørgrav * 9 Jun 1998 * * Incorporated into libfetch @@ -62,18 +60,15 @@ #define _GNU_SOURCE #endif -#include #include - #include -#include #include #include #include -#include -#include #include +#include +#include #include #include #include @@ -126,7 +121,7 @@ static int ftp_cmd(conn_t *, const char *, ...) LIBFETCH_PRINTFLIKE(2, 3); * Translate IPv4 mapped IPv6 address to IPv4 address */ static void -unmappedaddr(struct sockaddr_in6 *sin6, socklen_t *len) +unmappedaddr(struct sockaddr_in6 *sin6) { struct sockaddr_in *sin4; uint32_t addr; @@ -136,17 +131,12 @@ unmappedaddr(struct sockaddr_in6 *sin6, socklen_t *len) !IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) return; sin4 = (struct sockaddr_in *)sin6; -#ifdef s6_addr32 - addr = sin6->sin6_addr.s6_addr32[3]; -#else - memcpy(&addr, &sin6->sin6_addr.s6_addr[12], sizeof(addr)); -#endif + addr = *(u_int32_t *)(uintptr_t)&sin6->sin6_addr.s6_addr[12]; port = sin6->sin6_port; memset(sin4, 0, sizeof(struct sockaddr_in)); sin4->sin_addr.s_addr = addr; sin4->sin_port = port; sin4->sin_family = AF_INET; - *len = sizeof(struct sockaddr_in); #ifndef __linux__ sin4->sin_len = sizeof(struct sockaddr_in); #endif @@ -210,7 +200,7 @@ ftp_cmd(conn_t *conn, const char *fmt, ...) return (-1); } - r = fetch_write(conn, msg, len); + r = fetch_putln(conn, msg, len); free(msg); if (r == -1) { @@ -225,11 +215,11 @@ ftp_cmd(conn_t *conn, const char *fmt, ...) * Return a pointer to the filename part of a path */ static const char * -ftp_filename(const char *file, int *len, int *type, int subdir) +ftp_filename(const char *file, int *len, int *type) { const char *s; - if ((s = strrchr(file, '/')) == NULL || subdir) + if ((s = strrchr(file, '/')) == NULL) s = file; else s = s + 1; @@ -248,7 +238,7 @@ ftp_filename(const char *file, int *len, int *type, int subdir) * command. */ static int -ftp_pwd(conn_t *conn, char **pwd) +ftp_pwd(conn_t *conn, char *pwd, size_t pwdlen) { char *src, *dst, *end; int q; @@ -260,10 +250,7 @@ ftp_pwd(conn_t *conn, char **pwd) src = conn->buf + 4; if (src >= end || *src++ != '"') return (FTP_PROTOCOL_ERROR); - *pwd = malloc(end - src + 1); - if (*pwd == NULL) - return (FTP_PROTOCOL_ERROR); - for (q = 0, dst = *pwd; src < end; ++src) { + for (q = 0, dst = pwd; src < end && pwdlen--; ++src) { if (!q && *src == '"') q = 1; else if (q && *src != '"') @@ -273,12 +260,12 @@ ftp_pwd(conn_t *conn, char **pwd) else *dst++ = *src; } - *dst = '\0'; - if (**pwd != '/') { - free(*pwd); - *pwd = NULL; + if (!pwdlen) return (FTP_PROTOCOL_ERROR); - } + *dst = '\0'; +#if 0 + DEBUGF("pwd: [%s]\n", pwd); +#endif return (FTP_OK); } @@ -287,110 +274,69 @@ ftp_pwd(conn_t *conn, char **pwd) * file. */ static int -ftp_cwd(conn_t *conn, const char *path, int subdir) +ftp_cwd(conn_t *conn, const char *file) { const char *beg, *end; - char *pwd, *dst; + char pwd[PATH_MAX]; int e, i, len; - if (*path != '/') { - ftp_seterr(501); - return (-1); - } - ++path; - - /* Simple case: still in the home directory and no directory change. */ - if (conn->ftp_home == NULL && strchr(path, '/') == NULL && - (!subdir || *path == '\0')) - return 0; - - if ((e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || - (e = ftp_pwd(conn, &pwd)) != FTP_OK) { + /* If no slashes in name, no need to change dirs. */ + if ((end = strrchr(file, '/')) == NULL) + return (0); + if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); return (-1); } - if (conn->ftp_home == NULL && (conn->ftp_home = strdup(pwd)) == NULL) { - fetch_syserr(); - free(pwd); - return (-1); - } - if (*path == '/') { - while (path[1] == '/') - ++path; - dst = strdup(path); - } else if (strcmp(conn->ftp_home, "/") == 0) { - dst = strdup(path - 1); - } else { - if (asprintf(&dst, "%s/%s", conn->ftp_home, path) == -1) - dst = NULL; - } - if (dst == NULL) { - fetch_syserr(); - free(pwd); - return (-1); - } - - if (subdir) - end = dst + strlen(dst); - else - end = strrchr(dst, '/'); - for (;;) { len = strlen(pwd); /* Look for a common prefix between PWD and dir to fetch. */ - for (i = 0; i < len && i < end - dst; ++i) - if (pwd[i] != dst[i]) + for (i = 0; i <= len && i <= end - file; ++i) + if (pwd[i] != file[i]) break; +#if 0 + DEBUGF("have: [%.*s|%s]\n", i, pwd, pwd + i); + DEBUGF("want: [%.*s|%s]\n", i, file, file + i); +#endif /* Keep going up a dir until we have a matching prefix. */ - if (strcmp(pwd, "/") == 0) + if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/')) break; - if (pwd[i] == '\0' && (dst[i - 1] == '/' || dst[i] == '/')) - break; - free(pwd); - if ((e = ftp_cmd(conn, "CDUP\r\n")) != FTP_FILE_ACTION_OK || - (e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || - (e = ftp_pwd(conn, &pwd)) != FTP_OK) { + if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK || + (e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); - free(dst); return (-1); } } - free(pwd); #ifdef FTP_COMBINE_CWDS /* Skip leading slashes, even "////". */ - for (beg = dst + i; beg < end && *beg == '/'; ++beg, ++i) + for (beg = file + i; beg < end && *beg == '/'; ++beg, ++i) /* nothing */ ; /* If there is no trailing dir, we're already there. */ - if (beg >= end) { - free(dst); + if (beg >= end) return (0); - } /* Change to the directory all in one chunk (e.g., foo/bar/baz). */ - e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(end - beg), beg); - if (e == FTP_FILE_ACTION_OK) { - free(dst); + e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg); + if (e == FTP_FILE_ACTION_OK) return (0); - } #endif /* FTP_COMBINE_CWDS */ /* That didn't work so go back to legacy behavior (multiple CWDs). */ - for (beg = dst + i; beg < end; beg = dst + i + 1) { + for (beg = file + i; beg < end; beg = file + i + 1) { while (*beg == '/') ++beg, ++i; - for (++i; dst + i < end && dst[i] != '/'; ++i) + for (++i; file + i < end && file[i] != '/'; ++i) /* nothing */ ; - e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(dst + i - beg), beg); + e = ftp_cmd(conn, "CWD %.*s", (int)(file + i - beg), beg); if (e != FTP_FILE_ACTION_OK) { - free(dst); ftp_seterr(e); return (-1); } } - free(dst); return (0); } @@ -411,7 +357,7 @@ ftp_mode_type(conn_t *conn, int mode, int type) default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "MODE %c\r\n", mode)) != FTP_OK) { + if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) { if (mode == 'S') { /* * Stream mode is supposed to be the default - so @@ -440,10 +386,15 @@ ftp_mode_type(conn_t *conn, int mode, int type) type = 'A'; case 'A': break; + case 'd': + type = 'D'; + /* fallthrough */ + case 'D': + /* can't handle yet */ default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "TYPE %c\r\n", type)) != FTP_OK) + if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK) return (e); return (FTP_OK); @@ -457,7 +408,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) { char *ln; const char *filename; - int filenamelen, type, year; + int filenamelen, type; struct tm tm; time_t t; int e; @@ -465,14 +416,14 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) us->size = -1; us->atime = us->mtime = 0; - filename = ftp_filename(file, &filenamelen, &type, 0); + filename = ftp_filename(file, &filenamelen, &type); if ((e = ftp_mode_type(conn, 0, type)) != FTP_OK) { ftp_seterr(e); return (-1); } - e = ftp_cmd(conn, "SIZE %.*s\r\n", (int)filenamelen, filename); + e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -488,8 +439,9 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) } if (us->size == 0) us->size = -1; + DEBUGF("size: [%lld]\n", (long long)us->size); - e = ftp_cmd(conn, "MDTM %.*s\r\n", (int)filenamelen, filename); + e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -509,20 +461,22 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) return (-1); } if (sscanf(ln, "%04d%02d%02d%02d%02d%02d", - &year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) { ftp_seterr(FTP_PROTOCOL_ERROR); return (-1); } tm.tm_mon--; - tm.tm_year = year - 1900; + tm.tm_year -= 1900; tm.tm_isdst = -1; t = timegm(&tm); if (t == (time_t)-1) t = time(NULL); us->mtime = t; us->atime = t; - + DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec); return (0); } @@ -539,7 +493,10 @@ struct ftpio { static ssize_t ftp_readfn(void *, void *, size_t); static ssize_t ftp_writefn(void *, const void *, size_t); -static void ftp_closefn(void *); +#if 0 +static off_t ftp_seekfn(void *, off_t, int); +#endif +static int ftp_closefn(void *); static ssize_t ftp_readfn(void *v, void *buf, size_t len) @@ -601,36 +558,49 @@ ftp_writefn(void *v, const void *buf, size_t len) return (-1); } -static int -ftp_disconnect(conn_t *conn) +#if 0 +static off_t +ftp_seekfn(void *v, off_t pos __unused, int whence __unused) { - ftp_cmd(conn, "QUIT\r\n"); - return fetch_close(conn); + struct ftpio *io; + + io = (struct ftpio *)v; + if (io == NULL) { + errno = EBADF; + return ((off_t)-1); + } + errno = ESPIPE; + return ((off_t)-1); } +#endif -static void +static int ftp_disconnect(conn_t *conn); + +static int ftp_closefn(void *v) { struct ftpio *io; + int r; io = (struct ftpio *)v; if (io == NULL) { errno = EBADF; - return; + return (-1); } if (io->dir == -1) - return; + return (0); if (io->cconn == NULL || io->dconn == NULL) { errno = EBADF; - return; + return (-1); } fetch_close(io->dconn); - io->dconn = NULL; io->dir = -1; - ftp_chkerr(io->cconn); + io->dconn = NULL; + DEBUGF("Waiting for final status\n"); + r = ftp_chkerr(io->cconn); fetch_cache_put(io->cconn, ftp_disconnect); free(io); - return; + return (r == FTP_TRANSFER_COMPLETE) ? 0 : -1; } static fetchIO * @@ -657,18 +627,18 @@ ftp_setup(conn_t *cconn, conn_t *dconn, int mode) * Transfer file */ static fetchIO * -ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_arg, +ftp_transfer(conn_t *conn, const char *oper, const char *file, int mode, off_t offset, const char *flags) { - union anonymous { - struct sockaddr_storage ss; - struct sockaddr sa; - struct sockaddr_in6 sin6; - struct sockaddr_in sin4; - } u; + struct sockaddr_storage sa; + struct sockaddr_in6 *sin6; + struct sockaddr_in *sin4; const char *bindaddr; const char *filename; int filenamelen, type; +#if defined(IPV6_PORTRANGE) || defined(IP_PORTRANGE) + int low; +#endif int pasv, verbose; int e, sd = -1; socklen_t l; @@ -676,32 +646,32 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar fetchIO *df; /* check flags */ - pasv = !CHECK_FLAG('a'); +#if defined(IPV6_PORTRANGE) || defined(IP_PORTRANGE) + low = CHECK_FLAG('l'); +#endif + pasv = CHECK_FLAG('p') || !CHECK_FLAG('P'); verbose = CHECK_FLAG('v'); /* passive mode */ - if (!pasv) - pasv = ((s = getenv("FTP_PASSIVE_MODE")) != NULL && - strncasecmp(s, "no", 2) != 0); + if ((s = getenv("FTP_PASSIVE_MODE")) != NULL) + pasv = (strncasecmp(s, "no", 2) != 0); /* isolate filename */ - filename = ftp_filename(file, &filenamelen, &type, op_arg != NULL); + filename = ftp_filename(file, &filenamelen, &type); /* set transfer mode and data type */ if ((e = ftp_mode_type(conn, 0, type)) != FTP_OK) goto ouch; /* find our own address, bind, and listen */ - l = sizeof(u.ss); - if (getsockname(conn->sd, &u.sa, &l) == -1) + l = sizeof(sa); + if (getsockname(conn->sd, (struct sockaddr *)&sa, &l) == -1) goto sysouch; - if (u.ss.ss_family == AF_INET6) - unmappedaddr(&u.sin6, &l); - -retry_mode: + if (sa.ss_family == AF_INET6) + unmappedaddr((struct sockaddr_in6 *)&sa); /* open data socket */ - if ((sd = socket(u.ss.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sd = socket(sa.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { fetch_syserr(); return (NULL); } @@ -715,16 +685,16 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar /* send PASV command */ if (verbose) fetch_info("setting passive mode"); - switch (u.ss.ss_family) { + switch (sa.ss_family) { case AF_INET: - if ((e = ftp_cmd(conn, "PASV\r\n")) != FTP_PASSIVE_MODE) + if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE) goto ouch; break; case AF_INET6: - if ((e = ftp_cmd(conn, "EPSV\r\n")) != FTP_EPASSIVE_MODE) { + if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) { if (e == -1) goto ouch; - if ((e = ftp_cmd(conn, "LPSV\r\n")) != + if ((e = ftp_cmd(conn, "LPSV")) != FTP_LPASSIVE_MODE) goto ouch; } @@ -772,45 +742,40 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar goto ouch; } break; - case FTP_SYNTAX_ERROR: - if (verbose) - fetch_info("passive mode failed"); - /* Close socket and retry with passive mode. */ - pasv = 0; - close(sd); - sd = -1; - goto retry_mode; } /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %lu\r\n", (unsigned long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %lu", (unsigned long)offset) != FTP_FILE_OK) goto sysouch; /* construct sockaddr for data socket */ - l = sizeof(u.ss); - if (getpeername(conn->sd, &u.sa, &l) == -1) + l = sizeof(sa); + if (getpeername(conn->sd, (struct sockaddr *)&sa, &l) == -1) goto sysouch; - if (u.ss.ss_family == AF_INET6) - unmappedaddr(&u.sin6, &l); - switch (u.ss.ss_family) { + if (sa.ss_family == AF_INET6) + unmappedaddr((struct sockaddr_in6 *)&sa); + switch (sa.ss_family) { case AF_INET6: + sin6 = (struct sockaddr_in6 *)&sa; if (e == FTP_EPASSIVE_MODE) - u.sin6.sin6_port = htons(port); + sin6->sin6_port = htons(port); else { - memcpy(&u.sin6.sin6_addr, addr + 2, 16); - memcpy(&u.sin6.sin6_port, addr + 19, 2); + memcpy(&sin6->sin6_addr, addr + 2, 16); + memcpy(&sin6->sin6_port, addr + 19, 2); } break; case AF_INET: + sin4 = (struct sockaddr_in *)&sa; if (e == FTP_EPASSIVE_MODE) - u.sin4.sin_port = htons(port); + sin4->sin_port = htons(port); else { - memcpy(&u.sin4.sin_addr, addr, 4); - memcpy(&u.sin4.sin_port, addr + 4, 2); + memcpy(&sin4->sin_addr, addr, 4); + memcpy(&sin4->sin_port, addr + 4, 2); } break; default: + e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */ break; } @@ -819,19 +784,15 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar fetch_info("opening data connection"); bindaddr = getenv("FETCH_BIND_ADDRESS"); if (bindaddr != NULL && *bindaddr != '\0' && - fetch_bind(sd, u.ss.ss_family, bindaddr) != 0) - goto sysouch; - if (connect(sd, &u.sa, l) == -1) + (e = fetch_bind(sd, sa.ss_family, bindaddr)) != 0) + goto ouch; + if (connect(sd, (struct sockaddr *)&sa, l) == -1) goto sysouch; /* make the server initiate the transfer */ if (verbose) fetch_info("initiating transfer"); - if (op_arg) - e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); - else - e = ftp_cmd(conn, "%s %.*s\r\n", oper, - (int)filenamelen, filename); + e = ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -840,14 +801,14 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar uint16_t p; #if defined(IPV6_PORTRANGE) || defined(IP_PORTRANGE) int arg; - int low = CHECK_FLAG('l'); #endif int d; + char *ap; char hname[INET6_ADDRSTRLEN]; - switch (u.ss.ss_family) { + switch (sa.ss_family) { case AF_INET6: - u.sin6.sin6_port = 0; + ((struct sockaddr_in6 *)&sa)->sin6_port = 0; #ifdef IPV6_PORTRANGE arg = low ? IPV6_PORTRANGE_DEFAULT : IPV6_PORTRANGE_HIGH; if (setsockopt(sd, IPPROTO_IPV6, IPV6_PORTRANGE, @@ -856,7 +817,7 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar #endif break; case AF_INET: - u.sin4.sin_port = 0; + ((struct sockaddr_in *)&sa)->sin_port = 0; #ifdef IP_PORTRANGE arg = low ? IP_PORTRANGE_DEFAULT : IP_PORTRANGE_HIGH; if (setsockopt(sd, IPPROTO_IP, IP_PORTRANGE, @@ -867,50 +828,49 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar } if (verbose) fetch_info("binding data socket"); - if (bind(sd, &u.sa, l) == -1) + if (bind(sd, (struct sockaddr *)&sa, l) == -1) goto sysouch; if (listen(sd, 1) == -1) goto sysouch; /* find what port we're on and tell the server */ - if (getsockname(sd, &u.sa, &l) == -1) + if (getsockname(sd, (struct sockaddr *)&sa, &l) == -1) goto sysouch; - switch (u.ss.ss_family) { + switch (sa.ss_family) { case AF_INET: - a = ntohl(u.sin4.sin_addr.s_addr); - p = ntohs(u.sin4.sin_port); - e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d\r\n", + sin4 = (struct sockaddr_in *)&sa; + a = ntohl(sin4->sin_addr.s_addr); + p = ntohs(sin4->sin_port); + e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d", (a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff, (p >> 8) & 0xff, p & 0xff); break; case AF_INET6: +#define UC(b) (((int)b)&0xff) e = -1; - u.sin6.sin6_scope_id = 0; - if (getnameinfo(&u.sa, l, + sin6 = (struct sockaddr_in6 *)&sa; + sin6->sin6_scope_id = 0; + if (getnameinfo((struct sockaddr *)&sa, l, hname, sizeof(hname), NULL, 0, NI_NUMERICHOST) == 0) { - e = ftp_cmd(conn, "EPRT |%d|%s|%d|\r\n", 2, hname, - htons(u.sin6.sin6_port)); + e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname, + htons(sin6->sin6_port)); if (e == -1) goto ouch; } if (e != FTP_OK) { - unsigned char *ap = (void *)&u.sin6.sin6_addr.s6_addr; - uint16_t port = ntohs(u.sin6.sin6_port); + ap = (char *)&sin6->sin6_addr; e = ftp_cmd(conn, - "LPRT %d,%d,%u,%u,%u,%u,%u,%u,%u,%u," - "%u,%u,%u,%u,%u,%u,%u,%u,%d,%d,%d\r\n", + "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", 6, 16, - (unsigned)ap[0], (unsigned)ap[1], - (unsigned)ap[2], (unsigned)ap[3], - (unsigned)ap[4], (unsigned)ap[5], - (unsigned)ap[6], (unsigned)ap[7], - (unsigned)ap[8], (unsigned)ap[9], - (unsigned)ap[10], (unsigned)ap[11], - (unsigned)ap[12], (unsigned)ap[13], - (unsigned)ap[14], (unsigned)ap[15], - 2, port >> 8, port & 0xff); + UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]), + UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]), + UC(ap[8]), UC(ap[9]), UC(ap[10]), UC(ap[11]), + UC(ap[12]), UC(ap[13]), UC(ap[14]), UC(ap[15]), + 2, + (ntohs(sin6->sin6_port) >> 8) & 0xff, + ntohs(sin6->sin6_port) & 0xff); } break; default: @@ -922,17 +882,13 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %llu\r\n", (unsigned long long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %ju", (uintmax_t)offset) != FTP_FILE_OK) goto sysouch; /* make the server initiate the transfer */ if (verbose) fetch_info("initiating transfer"); - if (op_arg) - e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); - else - e = ftp_cmd(conn, "%s %.*s\r\n", oper, - (int)filenamelen, filename); + e = ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -967,7 +923,7 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_ar static int ftp_authenticate(conn_t *conn, struct url *url, struct url *purl) { - const char *user, *pwd, *login_name; + const char *user, *pwd, *logname; char pbuf[URL_USERLEN + 1 + URL_HOSTLEN + 1]; int e, len; @@ -978,34 +934,36 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl) fetch_netrc_auth(url); user = url->user; if (*user == '\0') - user = getenv("FTP_LOGIN"); + if ((user = getenv("FTP_LOGIN")) != NULL) + DEBUGF("FTP_LOGIN=%s\n", user); if (user == NULL || *user == '\0') user = FTP_ANONYMOUS_USER; if (purl && url->port == fetch_default_port(url->scheme)) - e = ftp_cmd(conn, "USER %s@%s\r\n", user, url->host); + e = ftp_cmd(conn, "USER %s@%s", user, url->host); else if (purl) - e = ftp_cmd(conn, "USER %s@%s@%d\r\n", user, url->host, url->port); + e = ftp_cmd(conn, "USER %s@%s@%d", user, url->host, url->port); else - e = ftp_cmd(conn, "USER %s\r\n", user); + e = ftp_cmd(conn, "USER %s", user); /* did the server request a password? */ if (e == FTP_NEED_PASSWORD) { pwd = url->pwd; if (*pwd == '\0') - pwd = getenv("FTP_PASSWORD"); + if ((pwd = getenv("FTP_PASSWORD")) != NULL) + DEBUGF("FTP_PASSWORD=%s\n", pwd); if (pwd == NULL || *pwd == '\0') { - if ((login_name = getlogin()) == 0) - login_name = FTP_ANONYMOUS_USER; - if ((len = snprintf(pbuf, URL_USERLEN + 2, "%s@", login_name)) < 0) + if ((logname = getlogin()) == NULL) + logname = FTP_ANONYMOUS_USER; + if ((len = snprintf(pbuf, URL_USERLEN + 1, "%s@", logname)) < 0) len = 0; - else if (len > URL_USERLEN + 1) - len = URL_USERLEN + 1; + else if (len > URL_USERLEN) + len = URL_USERLEN; gethostname(pbuf + len, sizeof(pbuf) - len); /* MAXHOSTNAMELEN can differ from URL_HOSTLEN + 1 */ pbuf[sizeof(pbuf) - 1] = '\0'; pwd = pbuf; } - e = ftp_cmd(conn, "PASS %s\r\n", pwd); + e = ftp_cmd(conn, "PASS %s", pwd); } return (e); @@ -1083,6 +1041,17 @@ ftp_connect(struct url *url, struct url *purl, const char *flags) return (NULL); } +/* + * Disconnect from server + */ +static int +ftp_disconnect(conn_t *conn) +{ + ftp_cmd(conn, "QUIT\r\n"); + return fetch_close(conn); +} + + /* * Check the proxy settings */ @@ -1107,8 +1076,8 @@ ftp_get_proxy(struct url * url, const char *flags) } if (!purl->port) purl->port = fetch_default_proxy_port(purl->scheme); - if (strcasecmp(purl->scheme, SCHEME_FTP) == 0 || - strcasecmp(purl->scheme, SCHEME_HTTP) == 0) + if (strcmp(purl->scheme, SCHEME_FTP) == 0 || + strcmp(purl->scheme, SCHEME_HTTP) == 0) return (purl); fetchFreeURL(purl); } @@ -1119,17 +1088,16 @@ ftp_get_proxy(struct url * url, const char *flags) * Process an FTP request */ fetchIO * -ftp_request(struct url *url, const char *op, const char *op_arg, - struct url_stat *us, struct url *purl, const char *flags) +ftp_request(struct url *url, const char *op, struct url_stat *us, + struct url *purl, const char *flags) { - fetchIO *f; - char *path; conn_t *conn; - int if_modified_since, oflag; + int oflag, iflag; struct url_stat local_us; /* check if we should use HTTP instead */ - if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) { + if (purl && (strcmp(purl->scheme, SCHEME_HTTP) == 0 || + strcmp(purl->scheme, SCHEME_HTTPS) == 0)) { if (strcmp(op, "STAT") == 0) return (http_request(url, "HEAD", us, purl, flags)); else if (strcmp(op, "RETR") == 0) @@ -1147,36 +1115,22 @@ ftp_request(struct url *url, const char *op, const char *op_arg, if (conn == NULL) return (NULL); - if ((path = fetchUnquotePath(url)) == NULL) { - fetch_close(conn); - fetch_syserr(); - return NULL; - } - /* change directory */ - if (ftp_cwd(conn, path, op_arg != NULL) == -1) { - fetch_close(conn); - free(path); - return (NULL); - } + if (ftp_cwd(conn, url->doc) == -1) + goto errsock; - if_modified_since = CHECK_FLAG('i'); - if (if_modified_since && us == NULL) + iflag = CHECK_FLAG('i'); + if (iflag && us == NULL) us = &local_us; /* stat file */ - if (us && ftp_stat(conn, path, us) == -1 + if (us && ftp_stat(conn, url->doc, us) == -1 && fetchLastErrCode != FETCH_PROTO - && fetchLastErrCode != FETCH_UNAVAIL) { - fetch_close(conn); - free(path); - return (NULL); - } + && fetchLastErrCode != FETCH_UNAVAIL) + goto errsock; - if (if_modified_since && url->last_modified > 0 && - url->last_modified >= us->mtime) { + if (iflag && url->ims_time > 0 && url->ims_time >= us->mtime) { fetch_cache_put(conn, ftp_disconnect); - free(path); fetchLastErrCode = FETCH_UNCHANGED; snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); return NULL; @@ -1184,8 +1138,8 @@ ftp_request(struct url *url, const char *op, const char *op_arg, /* just a stat */ if (strcmp(op, "STAT") == 0) { - fetch_cache_put(conn, ftp_disconnect); - free(path); + --conn->ref; + ftp_disconnect(conn); return fetchIO_unopen(NULL, NULL, NULL, NULL); } if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0) @@ -1194,9 +1148,11 @@ ftp_request(struct url *url, const char *op, const char *op_arg, oflag = O_RDONLY; /* initiate the transfer */ - f = (ftp_transfer(conn, op, path, op_arg, oflag, url->offset, flags)); - free(path); - return f; + return (ftp_transfer(conn, op, url->doc, oflag, url->offset, flags)); + +errsock: + ftp_disconnect(conn); + return (NULL); } /* @@ -1205,7 +1161,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg, fetchIO * fetchXGetFTP(struct url *url, struct url_stat *us, const char *flags) { - return (ftp_request(url, "RETR", NULL, us, ftp_get_proxy(url, flags), flags)); + return (ftp_request(url, "RETR", us, ftp_get_proxy(url, flags), flags)); } /* @@ -1223,7 +1179,7 @@ fetchGetFTP(struct url *url, const char *flags) fetchIO * fetchPutFTP(struct url *url, const char *flags) { - return (ftp_request(url, CHECK_FLAG('a') ? "APPE" : "STOR", NULL, NULL, + return (ftp_request(url, CHECK_FLAG('a') ? "APPE" : "STOR", NULL, ftp_get_proxy(url, flags), flags)); } @@ -1235,61 +1191,14 @@ fetchStatFTP(struct url *url, struct url_stat *us, const char *flags) { fetchIO *f; - f = ftp_request(url, "STAT", NULL, us, ftp_get_proxy(url, flags), flags); + f = ftp_request(url, "STAT", us, ftp_get_proxy(url, flags), flags); if (f == NULL) return (-1); fetchIO_close(f); + /* + * When op is "STAT", ftp_request() will return either NULL or + * (FILE *)1, never a valid FILE *, so we mustn't fclose(f) before + * returning, as it would cause a segfault. + */ return (0); } - -/* - * List a directory - */ -int -fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const char *flags) -{ - fetchIO *f; - char buf[2 * PATH_MAX], *eol, *eos; - ssize_t len; - size_t cur_off; - int ret; - - /* XXX What about proxies? */ - if (pattern == NULL || strcmp(pattern, "*") == 0) - pattern = ""; - f = ftp_request(url, "NLST", pattern, NULL, ftp_get_proxy(url, flags), flags); - if (f == NULL) - return -1; - - cur_off = 0; - ret = 0; - - while ((len = fetchIO_read(f, buf + cur_off, sizeof(buf) - cur_off)) > 0) { - cur_off += len; - while ((eol = memchr(buf, '\n', cur_off)) != NULL) { - if (len == eol - buf) - break; - if (eol != buf) { - if (eol[-1] == '\r') - eos = eol - 1; - else - eos = eol; - *eos = '\0'; - ret = fetch_add_entry(ue, url, buf, 0); - if (ret) - break; - cur_off -= eol - buf + 1; - memmove(buf, eol + 1, cur_off); - } - } - if (ret) - break; - } - if (cur_off != 0 || len < 0) { - /* Not RFC conform, bail out. */ - fetchIO_close(f); - return -1; - } - fetchIO_close(f); - return ret; -} diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 395e7d8f3..7fca85c5d 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1,7 +1,7 @@ -/* $FreeBSD: rev 267127 $ */ -/* $NetBSD: http.c,v 1.37 2014/06/11 13:12:12 joerg Exp $ */ /*- - * Copyright (c) 2000-2014 Dag-Erling Smorgrav + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2000-2014 Dag-Erling Smørgrav * Copyright (c) 2003 Thomas Klausner * Copyright (c) 2008, 2009 Joerg Sonnenberger * All rights reserved. @@ -72,6 +72,7 @@ #include #include #include +#include #include #include #include @@ -79,11 +80,18 @@ #include #include +#ifdef WITH_SSL +#include +#define MD5Init(c) MD5_Init(c) +#define MD5Update(c, data, len) MD5_Update(c, data, len) +#define MD5Final(md, c) MD5_Final(md, c) +#else +#include +#endif + #include #include -#include - #include #include "fetch.h" @@ -111,10 +119,11 @@ #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ || (xyz) == HTTP_MOVED_TEMP \ || (xyz) == HTTP_TEMP_REDIRECT \ + || (xyz) == HTTP_PERM_REDIRECT \ || (xyz) == HTTP_USE_PROXY \ || (xyz) == HTTP_SEE_OTHER) -#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599) +#define HTTP_ERROR(xyz) ((xyz) >= 400 && (xyz) <= 599) static int http_cmd(conn_t *, const char *, ...) LIBFETCH_PRINTFLIKE(2, 3); @@ -129,12 +138,14 @@ struct httpio int keep_alive; /* keep-alive mode */ char *buf; /* chunk buffer */ size_t bufsize; /* size of chunk buffer */ - ssize_t buflen; /* amount of data currently in buffer */ - int bufpos; /* current read offset in buffer */ + size_t buflen; /* amount of data currently in buffer */ + size_t bufpos; /* current read offset in buffer */ int eof; /* end-of-file flag */ int error; /* error flag */ size_t chunksize; /* remaining size of current chunk */ - off_t contentlength; /* remaining size of the content */ +#ifndef NDEBUG + size_t total; +#endif }; /* @@ -165,13 +176,25 @@ http_new_chunk(struct httpio *io) } } +#ifndef NDEBUG + if (fetchDebug) { + io->total += io->chunksize; + if (io->chunksize == 0) + fprintf(stderr, "%s(): end of last chunk\n", __func__); + else + fprintf(stderr, "%s(): new chunk: %lu (%lu)\n", + __func__, (unsigned long)io->chunksize, + (unsigned long)io->total); + } +#endif + return (io->chunksize); } /* * Grow the input buffer to at least len bytes */ -static int +static inline int http_growbuf(struct httpio *io, size_t len) { char *tmp; @@ -189,54 +212,53 @@ http_growbuf(struct httpio *io, size_t len) /* * Fill the input buffer, do chunk decoding on the fly */ -static int +static ssize_t http_fillbuf(struct httpio *io, size_t len) { + ssize_t nbytes; + if (io->error) return (-1); if (io->eof) return (0); - if (io->contentlength >= 0 && (off_t)len > io->contentlength) - len = io->contentlength; - + /* not chunked: just fetch the requested amount */ if (io->chunked == 0) { if (http_growbuf(io, len) == -1) return (-1); - if ((io->buflen = fetch_read(io->conn, io->buf, len)) == -1) { - io->error = 1; + if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) { + io->error = errno; return (-1); } - if (io->contentlength) - io->contentlength -= io->buflen; + io->buflen = nbytes; io->bufpos = 0; return (io->buflen); } + /* chunked, but we ran out: get the next chunk header */ if (io->chunksize == 0) { switch (http_new_chunk(io)) { case -1: - io->error = 1; + io->error = EPROTO; return (-1); case 0: io->eof = 1; - if (fetch_getln(io->conn) == -1) - return (-1); return (0); } } + /* fetch the requested amount, but no more than the current chunk */ if (len > io->chunksize) len = io->chunksize; if (http_growbuf(io, len) == -1) return (-1); - if ((io->buflen = fetch_read(io->conn, io->buf, len)) == -1) { - io->error = 1; + if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) { + io->error = errno; return (-1); } - io->chunksize -= io->buflen; - if (io->contentlength >= 0) - io->contentlength -= io->buflen; + io->bufpos = 0; + io->buflen = nbytes; + io->chunksize -= nbytes; if (io->chunksize == 0) { char endl[2]; @@ -249,8 +271,6 @@ http_fillbuf(struct httpio *io, size_t len) return (-1); } - io->bufpos = 0; - return (io->buflen); } @@ -301,36 +321,39 @@ http_writefn(void *v, const void *buf, size_t len) /* * Close function */ -static void +static int http_closefn(void *v) { struct httpio *io = (struct httpio *)v; + int r; if (io->keep_alive) { int val; val = 0; setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, - sizeof(val)); - fetch_cache_put(io->conn, fetch_close); + sizeof(val)); + fetch_cache_put(io->conn, fetch_close); #ifdef TCP_NOPUSH val = 1; setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val)); #endif + r = 0; } else { - fetch_close(io->conn); + r = fetch_close(io->conn); } free(io->buf); free(io); + return (r); } /* * Wrap a file descriptor up */ static fetchIO * -http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength) +http_funopen(conn_t *conn, int chunked, int keep_alive) { struct httpio *io; fetchIO *f; @@ -341,7 +364,6 @@ http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength) } io->conn = conn; io->chunked = chunked; - io->contentlength = clength; io->keep_alive = keep_alive; f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn); if (f == NULL) { @@ -369,7 +391,8 @@ typedef enum { hdr_last_modified, hdr_location, hdr_transfer_encoding, - hdr_www_authenticate + hdr_www_authenticate, + hdr_proxy_authenticate, } hdr_t; /* Names of interesting headers */ @@ -384,6 +407,7 @@ static struct { { hdr_location, "Location" }, { hdr_transfer_encoding, "Transfer-Encoding" }, { hdr_www_authenticate, "WWW-Authenticate" }, + { hdr_proxy_authenticate, "Proxy-Authenticate" }, { hdr_unknown, NULL }, }; @@ -409,7 +433,7 @@ http_cmd(conn_t *conn, const char *fmt, ...) return (-1); } - r = fetch_write(conn, msg, len); + r = fetch_putln(conn, msg, len); free(msg); if (r == -1) { @@ -474,21 +498,114 @@ http_match(const char *str, const char *hdr) return (hdr); } + /* - * Get the next header and return the appropriate symbolic code. + * Get the next header and return the appropriate symbolic code. We + * need to read one line ahead for checking for a continuation line + * belonging to the current header (continuation lines start with + * white space). + * + * We get called with a fresh line already in the conn buffer, either + * from the previous http_next_header() invocation, or, the first + * time, from a fetch_getln() performed by our caller. + * + * This stops when we encounter an empty line (we dont read beyond the header + * area). + * + * Note that the "headerbuf" is just a place to return the result. Its + * contents are not used for the next call. This means that no cleanup + * is needed when ie doing another connection, just call the cleanup when + * fully done to deallocate memory. */ -static hdr_t -http_next_header(conn_t *conn, const char **p) + +/* Limit the max number of continuation lines to some reasonable value */ +#define HTTP_MAX_CONT_LINES 10 + +/* Place into which to build a header from one or several lines */ +typedef struct { + char *buf; /* buffer */ + size_t bufsize; /* buffer size */ + size_t buflen; /* length of buffer contents */ +} http_headerbuf_t; + +static void +init_http_headerbuf(http_headerbuf_t *buf) { - int i; + buf->buf = NULL; + buf->bufsize = 0; + buf->buflen = 0; +} - if (fetch_getln(conn) == -1) - return (hdr_syserror); - while (conn->buflen && isspace((unsigned char)conn->buf[conn->buflen - 1])) +static void +clean_http_headerbuf(http_headerbuf_t *buf) +{ + if (buf->buf) + free(buf->buf); + init_http_headerbuf(buf); +} + +/* Remove whitespace at the end of the buffer */ +static void +http_conn_trimright(conn_t *conn) +{ + while (conn->buflen && + isspace((unsigned char)conn->buf[conn->buflen - 1])) conn->buflen--; conn->buf[conn->buflen] = '\0'; +} + +static hdr_t +http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p) +{ + unsigned int i, len; + + /* + * Have to do the stripping here because of the first line. So + * it's done twice for the subsequent lines. No big deal + */ + http_conn_trimright(conn); if (conn->buflen == 0) return (hdr_end); + + /* Copy the line to the headerbuf */ + if (hbuf->bufsize < conn->buflen + 1) { + if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL) + return (hdr_syserror); + hbuf->bufsize = conn->buflen + 1; + } + strcpy(hbuf->buf, conn->buf); + hbuf->buflen = conn->buflen; + + /* + * Fetch possible continuation lines. Stop at 1st non-continuation + * and leave it in the conn buffer + */ + for (i = 0; i < HTTP_MAX_CONT_LINES; i++) { + if (fetch_getln(conn) == -1) + return (hdr_syserror); + + /* + * Note: we carry on the idea from the previous version + * that a pure whitespace line is equivalent to an empty + * one (so it's not continuation and will be handled when + * we are called next) + */ + http_conn_trimright(conn); + if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0]) + break; + + /* Got a continuation line. Concatenate to previous */ + len = hbuf->buflen + conn->buflen; + if (hbuf->bufsize < len + 1) { + len *= 2; + if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL) + return (hdr_syserror); + hbuf->bufsize = len + 1; + } + strcpy(hbuf->buf + hbuf->buflen, conn->buf); + hbuf->buflen += conn->buflen; + } + /* * We could check for malformed headers but we don't really care. * A valid header starts with a token immediately followed by a @@ -496,11 +613,292 @@ http_next_header(conn_t *conn, const char **p) * characters except "()<>@,;:\\\"{}". */ for (i = 0; hdr_names[i].num != hdr_unknown; i++) - if ((*p = http_match(hdr_names[i].name, conn->buf)) != NULL) + if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL) return (hdr_names[i].num); + return (hdr_unknown); } +/************************** + * [Proxy-]Authenticate header parsing + */ + +/* + * Read doublequote-delimited string into output buffer obuf (allocated + * by caller, whose responsibility it is to ensure that it's big enough) + * cp points to the first char after the initial '"' + * Handles \ quoting + * Returns pointer to the first char after the terminating double quote, or + * NULL for error. + */ +static const char * +http_parse_headerstring(const char *cp, char *obuf) +{ + for (;;) { + switch (*cp) { + case 0: /* Unterminated string */ + *obuf = 0; + return (NULL); + case '"': /* Ending quote */ + *obuf = 0; + return (++cp); + case '\\': + if (*++cp == 0) { + *obuf = 0; + return (NULL); + } + /* FALLTHROUGH */ + default: + *obuf++ = *cp++; + } + } +} + +/* Http auth challenge schemes */ +typedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t; + +/* Data holder for a Basic or Digest challenge. */ +typedef struct { + http_auth_schemes_t scheme; + char *realm; + char *qop; + char *nonce; + char *opaque; + char *algo; + int stale; + int nc; /* Nonce count */ +} http_auth_challenge_t; + +static void +init_http_auth_challenge(http_auth_challenge_t *b) +{ + b->scheme = HTTPAS_UNKNOWN; + b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL; + b->stale = b->nc = 0; +} + +static void +clean_http_auth_challenge(http_auth_challenge_t *b) +{ + if (b->realm) + free(b->realm); + if (b->qop) + free(b->qop); + if (b->nonce) + free(b->nonce); + if (b->opaque) + free(b->opaque); + if (b->algo) + free(b->algo); + init_http_auth_challenge(b); +} + +/* Data holder for an array of challenges offered in an http response. */ +#define MAX_CHALLENGES 10 +typedef struct { + http_auth_challenge_t *challenges[MAX_CHALLENGES]; + int count; /* Number of parsed challenges in the array */ + int valid; /* We did parse an authenticate header */ +} http_auth_challenges_t; + +static void +init_http_auth_challenges(http_auth_challenges_t *cs) +{ + int i; + for (i = 0; i < MAX_CHALLENGES; i++) + cs->challenges[i] = NULL; + cs->count = cs->valid = 0; +} + +static void +clean_http_auth_challenges(http_auth_challenges_t *cs) +{ + int i; + /* We rely on non-zero pointers being allocated, not on the count */ + for (i = 0; i < MAX_CHALLENGES; i++) { + if (cs->challenges[i] != NULL) { + clean_http_auth_challenge(cs->challenges[i]); + free(cs->challenges[i]); + } + } + init_http_auth_challenges(cs); +} + +/* + * Enumeration for lexical elements. Separators will be returned as their own + * ascii value + */ +typedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258, + HTTPHL_ERROR = 259} http_header_lex_t; + +/* + * Determine what kind of token comes next and return possible value + * in buf, which is supposed to have been allocated big enough by + * caller. Advance input pointer and return element type. + */ +static int +http_header_lex(const char **cpp, char *buf) +{ + size_t l; + /* Eat initial whitespace */ + *cpp += strspn(*cpp, " \t"); + if (**cpp == 0) + return (HTTPHL_END); + + /* Separator ? */ + if (**cpp == ',' || **cpp == '=') + return (*((*cpp)++)); + + /* String ? */ + if (**cpp == '"') { + *cpp = http_parse_headerstring(++*cpp, buf); + if (*cpp == NULL) + return (HTTPHL_ERROR); + return (HTTPHL_STRING); + } + + /* Read other token, until separator or whitespace */ + l = strcspn(*cpp, " \t,="); + memcpy(buf, *cpp, l); + buf[l] = 0; + *cpp += l; + return (HTTPHL_WORD); +} + +/* + * Read challenges from http xxx-authenticate header and accumulate them + * in the challenges list structure. + * + * Headers with multiple challenges are specified by rfc2617, but + * servers (ie: squid) often send them in separate headers instead, + * which in turn is forbidden by the http spec (multiple headers with + * the same name are only allowed for pure comma-separated lists, see + * rfc2616 sec 4.2). + * + * We support both approaches anyway + */ +static int +http_parse_authenticate(const char *cp, http_auth_challenges_t *cs) +{ + int ret = -1; + http_header_lex_t lex; + char *key = malloc(strlen(cp) + 1); + char *value = malloc(strlen(cp) + 1); + char *buf = malloc(strlen(cp) + 1); + + if (key == NULL || value == NULL || buf == NULL) { + fetch_syserr(); + goto out; + } + + /* In any case we've seen the header and we set the valid bit */ + cs->valid = 1; + + /* Need word first */ + lex = http_header_lex(&cp, key); + if (lex != HTTPHL_WORD) + goto out; + + /* Loop on challenges */ + for (; cs->count < MAX_CHALLENGES; cs->count++) { + cs->challenges[cs->count] = + malloc(sizeof(http_auth_challenge_t)); + if (cs->challenges[cs->count] == NULL) { + fetch_syserr(); + goto out; + } + init_http_auth_challenge(cs->challenges[cs->count]); + if (strcasecmp(key, "basic") == 0) { + cs->challenges[cs->count]->scheme = HTTPAS_BASIC; + } else if (strcasecmp(key, "digest") == 0) { + cs->challenges[cs->count]->scheme = HTTPAS_DIGEST; + } else { + cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN; + /* + * Continue parsing as basic or digest may + * follow, and the syntax is the same for + * all. We'll just ignore this one when + * looking at the list + */ + } + + /* Loop on attributes */ + for (;;) { + /* Key */ + lex = http_header_lex(&cp, key); + if (lex != HTTPHL_WORD) + goto out; + + /* Equal sign */ + lex = http_header_lex(&cp, buf); + if (lex != '=') + goto out; + + /* Value */ + lex = http_header_lex(&cp, value); + if (lex != HTTPHL_WORD && lex != HTTPHL_STRING) + goto out; + + if (strcasecmp(key, "realm") == 0) { + cs->challenges[cs->count]->realm = + strdup(value); + } else if (strcasecmp(key, "qop") == 0) { + cs->challenges[cs->count]->qop = + strdup(value); + } else if (strcasecmp(key, "nonce") == 0) { + cs->challenges[cs->count]->nonce = + strdup(value); + } else if (strcasecmp(key, "opaque") == 0) { + cs->challenges[cs->count]->opaque = + strdup(value); + } else if (strcasecmp(key, "algorithm") == 0) { + cs->challenges[cs->count]->algo = + strdup(value); + } else if (strcasecmp(key, "stale") == 0) { + cs->challenges[cs->count]->stale = + strcasecmp(value, "no"); + } else { + /* ignore unknown attributes */ + } + + /* Comma or Next challenge or End */ + lex = http_header_lex(&cp, key); + /* + * If we get a word here, this is the beginning of the + * next challenge. Break the attributes loop + */ + if (lex == HTTPHL_WORD) + break; + + if (lex == HTTPHL_END) { + /* End while looking for ',' is normal exit */ + cs->count++; + ret = 0; + goto out; + } + /* Anything else is an error */ + if (lex != ',') + goto out; + + } /* End attributes loop */ + } /* End challenge loop */ + + /* + * Challenges max count exceeded. This really can't happen + * with normal data, something's fishy -> error + */ + +out: + if (key) + free(key); + if (value) + free(value); + if (buf) + free(buf); + return (ret); +} + + /* * Parse a last-modified header */ @@ -524,6 +922,9 @@ http_parse_mtime(const char *p, time_t *mtime) setlocale(LC_TIME, locale); if (r == NULL) return (-1); + DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec); *mtime = timegm(&tm); return (0); } @@ -540,6 +941,7 @@ http_parse_length(const char *p, off_t *length) len = len * 10 + (*p - '0'); if (*p) return (-1); + DEBUGF("content length: [%lld]\n", (long long)len); *length = len; return (0); } @@ -572,10 +974,14 @@ http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size) len = len * 10 + *p - '0'; if (*p || len < last - first + 1) return (-1); - if (first == -1) + if (first == -1) { + DEBUGF("content range: [*/%lld]\n", (long long)len); *length = 0; - else + } else { + DEBUGF("content range: [%lld-%lld/%lld]\n", + (long long)first, (long long)last, (long long)len); *length = last - first + 1; + } *offset = first; *size = len; return (0); @@ -642,6 +1048,292 @@ http_base64(const char *src) return (str); } + +/* + * Extract authorization parameters from environment value. + * The value is like scheme:realm:user:pass + */ +typedef struct { + char *scheme; + char *realm; + char *user; + char *password; +} http_auth_params_t; + +static void +init_http_auth_params(http_auth_params_t *s) +{ + s->scheme = s->realm = s->user = s->password = NULL; +} + +static void +clean_http_auth_params(http_auth_params_t *s) +{ + if (s->scheme) + free(s->scheme); + if (s->realm) + free(s->realm); + if (s->user) + free(s->user); + if (s->password) + free(s->password); + init_http_auth_params(s); +} + +static int +http_authfromenv(const char *p, http_auth_params_t *parms) +{ + int ret = -1; + char *v, *ve; + char *str = strdup(p); + + if (str == NULL) { + fetch_syserr(); + return (-1); + } + v = str; + + if ((ve = strchr(v, ':')) == NULL) + goto out; + + *ve = 0; + if ((parms->scheme = strdup(v)) == NULL) { + fetch_syserr(); + goto out; + } + v = ve + 1; + + if ((ve = strchr(v, ':')) == NULL) + goto out; + + *ve = 0; + if ((parms->realm = strdup(v)) == NULL) { + fetch_syserr(); + goto out; + } + v = ve + 1; + + if ((ve = strchr(v, ':')) == NULL) + goto out; + + *ve = 0; + if ((parms->user = strdup(v)) == NULL) { + fetch_syserr(); + goto out; + } + v = ve + 1; + + + if ((parms->password = strdup(v)) == NULL) { + fetch_syserr(); + goto out; + } + ret = 0; +out: + if (ret == -1) + clean_http_auth_params(parms); + if (str) + free(str); + return (ret); +} + + +/* + * Digest response: the code to compute the digest is taken from the + * sample implementation in RFC2616 + */ +#define IN const +#define OUT + +#define HASHLEN 16 +typedef unsigned char HASH[HASHLEN]; +#define HASHHEXLEN 32 +typedef char HASHHEX[HASHHEXLEN+1]; + +static const char *hexchars = "0123456789abcdef"; +static void +CvtHex(IN HASH Bin, OUT HASHHEX Hex) +{ + unsigned short i; + unsigned char j; + + for (i = 0; i < HASHLEN; i++) { + j = (Bin[i] >> 4) & 0xf; + Hex[i*2] = hexchars[j]; + j = Bin[i] & 0xf; + Hex[i*2+1] = hexchars[j]; + } + Hex[HASHHEXLEN] = '\0'; +}; + +/* calculate H(A1) as per spec */ +static void +DigestCalcHA1( + IN char * pszAlg, + IN char * pszUserName, + IN char * pszRealm, + IN char * pszPassword, + IN char * pszNonce, + IN char * pszCNonce, + OUT HASHHEX SessionKey + ) +{ + MD5_CTX Md5Ctx; + HASH HA1; + + MD5Init(&Md5Ctx); + MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword)); + MD5Final(HA1, &Md5Ctx); + if (strcasecmp(pszAlg, "md5-sess") == 0) { + + MD5Init(&Md5Ctx); + MD5Update(&Md5Ctx, HA1, HASHLEN); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce)); + MD5Final(HA1, &Md5Ctx); + } + CvtHex(HA1, SessionKey); +} + +/* calculate request-digest/response-digest as per HTTP Digest spec */ +static void +DigestCalcResponse( + IN HASHHEX HA1, /* H(A1) */ + IN char * pszNonce, /* nonce from server */ + IN char * pszNonceCount, /* 8 hex digits */ + IN char * pszCNonce, /* client nonce */ + IN char * pszQop, /* qop-value: "", "auth", "auth-int" */ + IN char * pszMethod, /* method from the request */ + IN char * pszDigestUri, /* requested URL */ + IN HASHHEX HEntity, /* H(entity body) if qop="auth-int" */ + OUT HASHHEX Response /* request-digest or response-digest */ + ) +{ +#if 0 + DEBUGF("Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n", + HA1, pszNonce, pszQop, pszMethod, pszDigestUri); +#endif + MD5_CTX Md5Ctx; + HASH HA2; + HASH RespHash; + HASHHEX HA2Hex; + + // calculate H(A2) + MD5Init(&Md5Ctx); + MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri)); + if (strcasecmp(pszQop, "auth-int") == 0) { + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, HEntity, HASHHEXLEN); + } + MD5Final(HA2, &Md5Ctx); + CvtHex(HA2, HA2Hex); + + // calculate response + MD5Init(&Md5Ctx); + MD5Update(&Md5Ctx, HA1, HASHHEXLEN); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce)); + MD5Update(&Md5Ctx, ":", 1); + if (*pszQop) { + MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce)); + MD5Update(&Md5Ctx, ":", 1); + MD5Update(&Md5Ctx, pszQop, strlen(pszQop)); + MD5Update(&Md5Ctx, ":", 1); + } + MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN); + MD5Final(RespHash, &Md5Ctx); + CvtHex(RespHash, Response); +} + +/* + * Generate/Send a Digest authorization header + * This looks like: [Proxy-]Authorization: credentials + * + * credentials = "Digest" digest-response + * digest-response = 1#( username | realm | nonce | digest-uri + * | response | [ algorithm ] | [cnonce] | + * [opaque] | [message-qop] | + * [nonce-count] | [auth-param] ) + * username = "username" "=" username-value + * username-value = quoted-string + * digest-uri = "uri" "=" digest-uri-value + * digest-uri-value = request-uri ; As specified by HTTP/1.1 + * message-qop = "qop" "=" qop-value + * cnonce = "cnonce" "=" cnonce-value + * cnonce-value = nonce-value + * nonce-count = "nc" "=" nc-value + * nc-value = 8LHEX + * response = "response" "=" request-digest + * request-digest = <"> 32LHEX <"> + */ +static int +http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c, + http_auth_params_t *parms, struct url *url) +{ + HASHHEX HA1; + HASHHEX digest; + int r; + char noncecount[10]; + char cnonce[40]; + char *options = NULL; + + if (!c->realm || !c->nonce) { + DEBUGF("realm/nonce not set in challenge\n"); + return(-1); + } + if (!c->algo) + c->algo = strdup(""); + + if (asprintf(&options, "%s%s%s%s", + *c->algo? ",algorithm=" : "", c->algo, + c->opaque? ",opaque=" : "", c->opaque?c->opaque:"") < 0) + return (-1); + + if (!c->qop) { + c->qop = strdup(""); + *noncecount = 0; + *cnonce = 0; + } else { + c->nc++; + sprintf(noncecount, "%08x", c->nc); + /* We don't try very hard with the cnonce ... */ + sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0)); + } + + DigestCalcHA1(c->algo, parms->user, c->realm, + parms->password, c->nonce, cnonce, HA1); + DEBUGF("HA1: [%s]\n", HA1); + DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop, + "GET", url->doc, "", digest); + + if (c->qop[0]) { + r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\"," + "nonce=\"%s\",uri=\"%s\",response=\"%s\"," + "qop=\"auth\", cnonce=\"%s\", nc=%s%s", + hdr, parms->user, c->realm, + c->nonce, url->doc, digest, + cnonce, noncecount, options); + } else { + r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\"," + "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s", + hdr, parms->user, c->realm, + c->nonce, url->doc, digest, options); + } + if (options) + free(options); + return (r); +} + /* * Encode username and password */ @@ -651,68 +1343,65 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd) char *upw, *auth; int r; + DEBUGF("basic: usr: [%s]\n", usr); + DEBUGF("basic: pwd: [%s]\n", pwd); if (asprintf(&upw, "%s:%s", usr, pwd) == -1) return (-1); auth = http_base64(upw); free(upw); if (auth == NULL) return (-1); - r = http_cmd(conn, "%s: Basic %s\r\n", hdr, auth); + r = http_cmd(conn, "%s: Basic %s", hdr, auth); free(auth); return (r); } /* - * Send an authorization header + * Chose the challenge to answer and call the appropriate routine to + * produce the header. */ static int -http_authorize(conn_t *conn, const char *hdr, const char *p) +http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs, + http_auth_params_t *parms, struct url *url) { - /* basic authorization */ - if (strncasecmp(p, "basic:", 6) == 0) { - char *user, *pwd, *str; - int r; - - /* skip realm */ - for (p += 6; *p && *p != ':'; ++p) - /* nothing */ ; - if (!*p || strchr(++p, ':') == NULL) - return (-1); - if ((str = strdup(p)) == NULL) - return (-1); /* XXX */ - user = str; - pwd = strchr(str, ':'); - *pwd++ = '\0'; - r = http_basic_auth(conn, hdr, user, pwd); - free(str); - return (r); + http_auth_challenge_t *digest = NULL; + int i; + + /* If user or pass are null we're not happy */ + if (!parms->user || !parms->password) { + DEBUGF("NULL usr or pass\n"); + return (-1); + } + + /* Look for a Digest */ + for (i = 0; i < cs->count; i++) { + if (cs->challenges[i]->scheme == HTTPAS_DIGEST) + digest = cs->challenges[i]; } - return (-1); -} + /* Error if "Digest" was specified and there is no Digest challenge */ + if (!digest && + (parms->scheme && strcasecmp(parms->scheme, "digest") == 0)) { + DEBUGF("Digest auth in env, not supported by peer\n"); + return (-1); + } + /* + * If "basic" was specified in the environment, or there is no Digest + * challenge, do the basic thing. Don't need a challenge for this, + * so no need to check basic!=NULL + */ + if (!digest || + (parms->scheme && strcasecmp(parms->scheme, "basic") == 0)) + return (http_basic_auth(conn,hdr,parms->user,parms->password)); + + /* Else, prefer digest. We just checked that it's not NULL */ + return (http_digest_auth(conn, hdr, digest, parms, url)); +} /***************************************************************************** * Helper functions for connecting to a server or proxy */ -/* - * Send headers consumed by the proxy server. - */ -static void -send_proxy_headers(conn_t *conn, struct url *purl) -{ - char *p; - - /* proxy authorization */ - if (purl) { - if (*purl->user || *purl->pwd) - http_basic_auth(conn, "Proxy-Authorization", - purl->user, purl->pwd); - else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0') - http_authorize(conn, "Proxy-Authorization", p); - } -} - /* * Connect to the correct HTTP server or proxy. */ @@ -721,12 +1410,15 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) { struct url *curl; conn_t *conn; - const char *p; hdr_t h; - int af, verbose; + http_headerbuf_t headerbuf; + const char *p; + int verbose; + int af; #ifdef TCP_NOPUSH int val; #endif + int serrno; #ifdef INET6 af = AF_UNSPEC; @@ -752,38 +1444,37 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) if ((conn = fetch_connect(curl, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); - if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) { - http_cmd(conn, "CONNECT %s:%d HTTP/1.1\r\n", + init_http_headerbuf(&headerbuf); + if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) { + http_cmd(conn, "CONNECT %s:%d HTTP/1.1", URL->host, URL->port); - - send_proxy_headers(conn, purl); - - http_cmd(conn, "\r\n"); - + http_cmd(conn, "Host: %s:%d", + URL->host, URL->port); + http_cmd(conn, "%s", ""); if (http_get_reply(conn) != HTTP_OK) { http_seterr(conn->err); - fetch_close(conn); - return (NULL); + goto ouch; } /* Read and discard the rest of the proxy response */ + if (fetch_getln(conn) < 0) { + fetch_syserr(); + goto ouch; + } do { - switch ((h = http_next_header(conn, &p))) { + switch ((h = http_next_header(conn, &headerbuf, &p))) { case hdr_syserror: fetch_syserr(); - fetch_close(conn); - return (NULL); + goto ouch; case hdr_error: http_seterr(HTTP_PROTOCOL_ERROR); - fetch_close(conn); - return (NULL); + goto ouch; default: /* ignore */ ; } } while (h > hdr_end); } - if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && + if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && fetch_ssl(conn, URL, verbose) == -1) { - fetch_close(conn); /* grrr */ #ifdef EAUTH errno = EAUTH; @@ -791,7 +1482,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) errno = EPERM; #endif fetch_syserr(); - return (NULL); + goto ouch; } #ifdef TCP_NOPUSH @@ -799,7 +1490,14 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val)); #endif + clean_http_headerbuf(&headerbuf); return (conn); +ouch: + serrno = errno; + clean_http_headerbuf(&headerbuf); + fetch_close(conn); + errno = serrno; + return (NULL); } static struct url * @@ -818,32 +1516,27 @@ http_get_proxy(struct url * url, const char *flags) strcpy(purl->scheme, SCHEME_HTTP); if (!purl->port) purl->port = fetch_default_proxy_port(purl->scheme); - if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0) + if (strcmp(purl->scheme, SCHEME_HTTP) == 0) return (purl); fetchFreeURL(purl); } return (NULL); } -static void -set_if_modified_since(conn_t *conn, time_t last_modified) -{ - static const char weekdays[] = "SunMonTueWedThuFriSat"; - static const char months[] = "JanFebMarAprMayJunJulAugSepOctNovDec"; - struct tm tm; - char buf[80]; - gmtime_r(&last_modified, &tm); - snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4ld %02d:%02d:%02d GMT", - weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3, - (long)(tm.tm_year + 1900), tm.tm_hour, tm.tm_min, tm.tm_sec); - http_cmd(conn, "If-Modified-Since: %s\r\n", buf); -} /***************************************************************************** * Core */ +fetchIO * +http_request(struct url *URL, const char *op, struct url_stat *us, + struct url *purl, const char *flags) +{ + + return (http_request_body(URL, op, us, purl, flags, NULL, NULL)); +} + /* * Send a request and process the reply * @@ -851,25 +1544,36 @@ set_if_modified_since(conn_t *conn, time_t last_modified) * XXX off into a separate function. */ fetchIO * -http_request(struct url *URL, const char *op, struct url_stat *us, - struct url *purl, const char *flags) +http_request_body(struct url *URL, const char *op, struct url_stat *us, + struct url *purl, const char *flags, const char *content_type, + const char *body) { - conn_t *conn = NULL; + char timebuf[80]; + char hbuf[URL_HOSTLEN + 7], *host; + conn_t *conn; struct url *url, *new; - int chunked, direct, if_modified_since, need_auth, noredirect; - int keep_alive, verbose, cached; + int chunked, direct, ims, keep_alive, noredirect, verbose, cached; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; const char *p; fetchIO *f; hdr_t h; - char hbuf[URL_HOSTLEN + 7], *host; + struct tm *timestruct; + http_headerbuf_t headerbuf; + http_auth_challenges_t server_challenges; + http_auth_challenges_t proxy_challenges; + size_t body_len; + + /* The following calls don't allocate anything */ + init_http_headerbuf(&headerbuf); + init_http_auth_challenges(&server_challenges); + init_http_auth_challenges(&proxy_challenges); direct = CHECK_FLAG('d'); noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); - if_modified_since = CHECK_FLAG('i'); + ims = CHECK_FLAG('i'); keep_alive = 0; if (direct && purl) { @@ -880,12 +1584,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us, /* try the provided URL first */ url = URL; - /* if the A flag is set, we only get one try */ - n = noredirect ? 1 : MAX_REDIRECT; + n = MAX_REDIRECT; i = 0; e = HTTP_PROTOCOL_ERROR; - need_auth = 0; do { new = NULL; chunked = 0; @@ -903,96 +1605,144 @@ http_request(struct url *URL, const char *op, struct url_stat *us, /* were we redirected to an FTP URL? */ if (purl == NULL && strcmp(url->scheme, SCHEME_FTP) == 0) { if (strcmp(op, "GET") == 0) - return (ftp_request(url, "RETR", NULL, us, purl, flags)); + return (ftp_request(url, "RETR", us, purl, flags)); else if (strcmp(op, "HEAD") == 0) - return (ftp_request(url, "STAT", NULL, us, purl, flags)); + return (ftp_request(url, "STAT", us, purl, flags)); } /* connect to server or proxy */ - if (conn != NULL) - fetch_close(conn); - if ((conn = http_connect(url, purl, flags, &cached)) == NULL) goto ouch; + /* append port number only if necessary */ host = url->host; -#ifdef INET6 - if (strchr(url->host, ':')) { - snprintf(hbuf, sizeof(hbuf), "[%s]", url->host); - host = hbuf; - } -#endif if (url->port != fetch_default_port(url->scheme)) { - if (host != hbuf) { - strcpy(hbuf, host); - host = hbuf; - } - snprintf(hbuf + strlen(hbuf), - sizeof(hbuf) - strlen(hbuf), ":%d", url->port); + snprintf(hbuf, sizeof(hbuf), "%s:%d", host, url->port); + host = hbuf; } /* send request */ if (verbose) fetch_info("requesting %s://%s%s", url->scheme, host, url->doc); - if (purl && strcasecmp(url->scheme, SCHEME_HTTPS) != 0) { - http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n", + if (purl && strcmp(url->scheme, SCHEME_HTTPS) != 0) { + http_cmd(conn, "%s %s://%s%s HTTP/1.1", op, url->scheme, host, url->doc); } else { - http_cmd(conn, "%s %s HTTP/1.1\r\n", + http_cmd(conn, "%s %s HTTP/1.1", op, url->doc); } - if (if_modified_since && url->last_modified > 0) - set_if_modified_since(conn, url->last_modified); - + if (ims && url->ims_time) { + timestruct = gmtime((time_t *)&url->ims_time); + (void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT", + timestruct); + if (verbose) + fetch_info("If-Modified-Since: %s", timebuf); + http_cmd(conn, "If-Modified-Since: %s", timebuf); + } /* virtual host */ - http_cmd(conn, "Host: %s\r\n", host); - - if (strcasecmp(url->scheme, SCHEME_HTTPS) != 0) - send_proxy_headers(conn, purl); - - /* server authorization */ - if (need_auth || *url->user || *url->pwd) { - if (*url->user || *url->pwd) - http_basic_auth(conn, "Authorization", url->user, url->pwd); - else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0') - http_authorize(conn, "Authorization", p); - else if (fetchAuthMethod && fetchAuthMethod(url) == 0) { - http_basic_auth(conn, "Authorization", url->user, url->pwd); + http_cmd(conn, "Host: %s", host); + + /* + * Proxy authorization: we only send auth after we received + * a 407 error. We do not first try basic anyway (changed + * when support was added for digest-auth) + */ + if (purl && proxy_challenges.valid) { + http_auth_params_t aparams; + init_http_auth_params(&aparams); + if (*purl->user || *purl->pwd) { + aparams.user = strdup(purl->user); + aparams.password = strdup(purl->pwd); + } else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && + *p != '\0') { + if (http_authfromenv(p, &aparams) < 0) { + http_seterr(HTTP_NEED_PROXY_AUTH); + goto ouch; + } + } else if (fetch_netrc_auth(purl) == 0) { + aparams.user = strdup(purl->user); + aparams.password = strdup(purl->pwd); + } + http_authorize(conn, "Proxy-Authorization", + &proxy_challenges, &aparams, url); + clean_http_auth_params(&aparams); + } + + /* + * Server authorization: we never send "a priori" + * Basic auth, which used to be done if user/pass were + * set in the url. This would be weird because we'd send the + * password in the clear even if Digest is finally to be + * used (it would have made more sense for the + * pre-digest version to do this when Basic was specified + * in the environment) + */ + if (server_challenges.valid) { + http_auth_params_t aparams; + init_http_auth_params(&aparams); + if (*url->user || *url->pwd) { + aparams.user = strdup(url->user); + aparams.password = strdup(url->pwd); + } else if ((p = getenv("HTTP_AUTH")) != NULL && + *p != '\0') { + if (http_authfromenv(p, &aparams) < 0) { + http_seterr(HTTP_NEED_AUTH); + goto ouch; + } + } else if (fetch_netrc_auth(url) == 0) { + aparams.user = strdup(url->user); + aparams.password = strdup(url->pwd); + } else if (fetchAuthMethod && + fetchAuthMethod(url) == 0) { + aparams.user = strdup(url->user); + aparams.password = strdup(url->pwd); } else { http_seterr(HTTP_NEED_AUTH); goto ouch; } + http_authorize(conn, "Authorization", + &server_challenges, &aparams, url); + clean_http_auth_params(&aparams); } /* other headers */ + if ((p = getenv("HTTP_ACCEPT")) != NULL) { + if (*p != '\0') + http_cmd(conn, "Accept: %s", p); + } else { + http_cmd(conn, "Accept: */*"); + } if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') { if (strcasecmp(p, "auto") == 0) - http_cmd(conn, "Referer: %s://%s%s\r\n", + http_cmd(conn, "Referer: %s://%s%s", url->scheme, host, url->doc); else - http_cmd(conn, "Referer: %s\r\n", p); + http_cmd(conn, "Referer: %s", p); } if ((p = getenv("HTTP_USER_AGENT")) != NULL) { /* no User-Agent if defined but empty */ if (*p != '\0') - http_cmd(conn, "User-Agent: %s\r\n", p); + http_cmd(conn, "User-Agent: %s", p); } else { /* default User-Agent */ - http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER); + http_cmd(conn, "User-Agent: " _LIBFETCH_VER); } + if (url->offset > 0) + http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); - /* - * Some servers returns 406 (Not Acceptable) if the Accept field is not - * provided by the user agent, such example is http://alioth.debian.org. - */ - http_cmd(conn, "Accept: */*\r\n"); + if (body) { + body_len = strlen(body); + http_cmd(conn, "Content-Length: %zu", body_len); + if (content_type != NULL) + http_cmd(conn, "Content-Type: %s", content_type); + } - if (url->offset > 0) - http_cmd(conn, "Range: bytes=%lld-\r\n", (long long)url->offset); + http_cmd(conn, "%s", ""); - http_cmd(conn, "\r\n"); + if (body) + fetch_write(conn, body, body_len); /* * Force the queued request to be dispatched. Normally, one @@ -1019,16 +1769,17 @@ http_request(struct url *URL, const char *op, struct url_stat *us, break; case HTTP_MOVED_PERM: case HTTP_MOVED_TEMP: + case HTTP_TEMP_REDIRECT: + case HTTP_PERM_REDIRECT: case HTTP_SEE_OTHER: case HTTP_USE_PROXY: - case HTTP_TEMP_REDIRECT: /* * Not so fine, but we still have to read the * headers to get the new location. */ break; case HTTP_NEED_AUTH: - if (need_auth) { + if (server_challenges.valid) { /* * We already sent out authorization code, * so there's nothing more we can do. @@ -1041,13 +1792,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us, fetch_info("server requires authorization"); break; case HTTP_NEED_PROXY_AUTH: - /* - * If we're talking to a proxy, we already sent - * our proxy authorization code, so there's - * nothing more we can do. - */ - http_seterr(conn->err); - goto ouch; + if (proxy_challenges.valid) { + /* + * We already sent our proxy + * authorization code, so there's + * nothing more we can do. */ + http_seterr(conn->err); + goto ouch; + } + /* try again, but send the password this time */ + if (verbose) + fetch_info("proxy requires authorization"); + break; case HTTP_BAD_RANGE: /* * This can happen if we ask for 0 bytes because @@ -1058,9 +1814,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case HTTP_PROTOCOL_ERROR: /* fall through */ case -1: +#if 0 --i; if (cached) continue; +#endif fetch_syserr(); goto ouch; default: @@ -1070,9 +1828,13 @@ http_request(struct url *URL, const char *op, struct url_stat *us, /* fall through so we can get the full error message */ } - /* get headers */ + /* get headers. http_next_header expects one line readahead */ + if (fetch_getln(conn) == -1) { + fetch_syserr(); + goto ouch; + } do { - switch ((h = http_next_header(conn, &p))) { + switch ((h = http_next_header(conn, &headerbuf, &p))) { case hdr_syserror: fetch_syserr(); goto ouch; @@ -1095,26 +1857,43 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case hdr_location: if (!HTTP_REDIRECT(conn->err)) break; + /* + * if the A flag is set, we don't follow + * temporary redirects. + */ + if (noredirect && + conn->err != HTTP_MOVED_PERM && + conn->err != HTTP_PERM_REDIRECT && + conn->err != HTTP_USE_PROXY) { + n = 1; + break; + } if (new) free(new); if (verbose) - fetch_info("%d redirect to %s", conn->err, p); + fetch_info("%d redirect to %s", + conn->err, p); if (*p == '/') /* absolute path */ - new = fetchMakeURL(url->scheme, url->host, url->port, p, - url->user, url->pwd); + new = fetchMakeURL(url->scheme, url->host, + url->port, p, url->user, url->pwd); else new = fetchParseURL(p); if (new == NULL) { /* XXX should set an error code */ + DEBUGF("failed to parse new URL\n"); goto ouch; } - if (!*new->user && !*new->pwd) { + + /* Only copy credentials if the host matches */ + if (strcmp(new->host, url->host) == 0 && + !*new->user && !*new->pwd) { strcpy(new->user, url->user); strcpy(new->pwd, url->pwd); } new->offset = url->offset; new->length = url->length; + new->ims_time = url->ims_time; break; case hdr_transfer_encoding: /* XXX weak test*/ @@ -1123,7 +1902,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case hdr_www_authenticate: if (conn->err != HTTP_NEED_AUTH) break; - /* if we were smarter, we'd check the method and realm */ + if (http_parse_authenticate(p, &server_challenges) == 0) + ++n; + break; + case hdr_proxy_authenticate: + if (conn->err != HTTP_NEED_PROXY_AUTH) + break; + if (http_parse_authenticate(p, &proxy_challenges) == 0) + ++n; break; case hdr_end: /* fall through */ @@ -1134,9 +1920,17 @@ http_request(struct url *URL, const char *op, struct url_stat *us, } while (h > hdr_end); /* we need to provide authentication */ - if (conn->err == HTTP_NEED_AUTH) { + if (conn->err == HTTP_NEED_AUTH || + conn->err == HTTP_NEED_PROXY_AUTH) { e = conn->err; - need_auth = 1; + if ((conn->err == HTTP_NEED_AUTH && + !server_challenges.valid) || + (conn->err == HTTP_NEED_PROXY_AUTH && + !proxy_challenges.valid)) { + /* 401/7 but no www/proxy-authenticate ?? */ + DEBUGF("%03d without auth header\n", conn->err); + goto ouch; + } fetch_close(conn); conn = NULL; continue; @@ -1144,9 +1938,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us, /* requested range not satisfiable */ if (conn->err == HTTP_BAD_RANGE) { - if (url->offset == size && url->length == 0) { + if (url->offset > 0 && url->length == 0) { /* asked for 0 bytes; fake it */ offset = url->offset; + clength = -1; conn->err = HTTP_OK; break; } else { @@ -1156,19 +1951,21 @@ http_request(struct url *URL, const char *op, struct url_stat *us, } /* we have a hit or an error */ - if (conn->err == HTTP_OK || - conn->err == HTTP_PARTIAL || - conn->err == HTTP_NOT_MODIFIED || - HTTP_ERROR(conn->err)) + if (conn->err == HTTP_OK + || conn->err == HTTP_NOT_MODIFIED + || conn->err == HTTP_PARTIAL + || HTTP_ERROR(conn->err)) break; /* all other cases: we got a redirect */ e = conn->err; - need_auth = 0; + clean_http_auth_challenges(&server_challenges); fetch_close(conn); conn = NULL; - if (!new) + if (!new) { + DEBUGF("redirect with no new location\n"); break; + } if (url != URL) fetchFreeURL(url); url = new; @@ -1180,9 +1977,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us, goto ouch; } - /* fill in stats */ - if (us && size) { - us->size = size; + DEBUGF("offset %lld, length %lld, size %lld, clength %lld\n", + (long long)offset, (long long)length, + (long long)size, (long long)clength); + + if (conn->err == HTTP_NOT_MODIFIED) { + http_seterr(HTTP_NOT_MODIFIED); + if (keep_alive) { + fetch_cache_put(conn, fetch_close); + } else { + fetch_close(conn); + } + return (NULL); } /* check for inconsistencies */ @@ -1194,7 +2000,6 @@ http_request(struct url *URL, const char *op, struct url_stat *us, clength = length; if (clength != -1) length = offset + clength; - if (length != -1 && size != -1 && length != size) { http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; @@ -1218,20 +2023,8 @@ http_request(struct url *URL, const char *op, struct url_stat *us, URL->offset = offset; URL->length = clength; - if (clength == -1 && !chunked && conn->err != HTTP_NOT_MODIFIED) - keep_alive = 0; - - if (conn->err == HTTP_NOT_MODIFIED) { - http_seterr(HTTP_NOT_MODIFIED); - if (keep_alive) { - fetch_cache_put(conn, fetch_close); - conn = NULL; - } - goto ouch; - } - /* wrap it up in a fetchIO */ - if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) { + if ((f = http_funopen(conn, chunked, keep_alive)) == NULL) { fetch_syserr(); goto ouch; } @@ -1242,17 +2035,12 @@ http_request(struct url *URL, const char *op, struct url_stat *us, fetchFreeURL(purl); if (HTTP_ERROR(conn->err)) { - - if (keep_alive) { - char buf[512]; - do { - } while (fetchIO_read(f, buf, sizeof(buf)) > 0); - } - fetchIO_close(f); f = NULL; } - + clean_http_headerbuf(&headerbuf); + clean_http_auth_challenges(&server_challenges); + clean_http_auth_challenges(&proxy_challenges); return (f); ouch: @@ -1262,6 +2050,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us, fetchFreeURL(purl); if (conn != NULL) fetch_close(conn); + clean_http_headerbuf(&headerbuf); + clean_http_auth_challenges(&server_challenges); + clean_http_auth_challenges(&proxy_challenges); return (NULL); } @@ -1316,264 +2107,27 @@ fetchStatHTTP(struct url *URL, struct url_stat *us, const char *flags) return (0); } -enum http_states { - ST_NONE, - ST_LT, - ST_LTA, - ST_TAGA, - ST_H, - ST_R, - ST_E, - ST_F, - ST_HREF, - ST_HREFQ, - ST_TAG, - ST_TAGAX, - ST_TAGAQ -}; - -struct index_parser { - struct url_list *ue; - struct url *url; - enum http_states state; -}; - -static ssize_t -parse_index(struct index_parser *parser, const char *buf, size_t len) -{ - char *end_attr, p = *buf; - - switch (parser->state) { - case ST_NONE: - /* Plain text, not in markup */ - if (p == '<') - parser->state = ST_LT; - return 1; - case ST_LT: - /* In tag -- "<" already found */ - if (p == '>') - parser->state = ST_NONE; - else if (p == 'a' || p == 'A') - parser->state = ST_LTA; - else if (!isspace((unsigned char)p)) - parser->state = ST_TAG; - return 1; - case ST_LTA: - /* In tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (isspace((unsigned char)p)) - parser->state = ST_TAGA; - else - parser->state = ST_TAG; - return 1; - case ST_TAG: - /* In tag, but not "') - parser->state = ST_NONE; - return 1; - case ST_TAGA: - /* In a-tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (p == 'h' || p == 'H') - parser->state = ST_H; - else if (!isspace((unsigned char)p)) - parser->state = ST_TAGAX; - return 1; - case ST_TAGAX: - /* In unknown keyword in a-tag */ - if (p == '>') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (isspace((unsigned char)p)) - parser->state = ST_TAGA; - return 1; - case ST_TAGAQ: - /* In a-tag, unknown argument for keys. */ - if (p == '>') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGA; - return 1; - case ST_H: - /* In a-tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (p == 'r' || p == 'R') - parser->state = ST_R; - else if (isspace((unsigned char)p)) - parser->state = ST_TAGA; - else - parser->state = ST_TAGAX; - return 1; - case ST_R: - /* In a-tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (p == 'e' || p == 'E') - parser->state = ST_E; - else if (isspace((unsigned char)p)) - parser->state = ST_TAGA; - else - parser->state = ST_TAGAX; - return 1; - case ST_E: - /* In a-tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (p == 'f' || p == 'F') - parser->state = ST_F; - else if (isspace((unsigned char)p)) - parser->state = ST_TAGA; - else - parser->state = ST_TAGAX; - return 1; - case ST_F: - /* In a-tag -- "') - parser->state = ST_NONE; - else if (p == '"') - parser->state = ST_TAGAQ; - else if (p == '=') - parser->state = ST_HREF; - else if (!isspace((unsigned char)p)) - parser->state = ST_TAGAX; - return 1; - case ST_HREF: - /* In a-tag -- "state = ST_HREFQ; - else if (!isspace((unsigned char)p)) - parser->state = ST_TAGA; - return 1; - case ST_HREFQ: - /* In href of the a-tag */ - end_attr = memchr(buf, '"', len); - if (end_attr == NULL) - return 0; - *end_attr = '\0'; - parser->state = ST_TAGA; - if (fetch_add_entry(parser->ue, parser->url, buf, 1)) - return -1; - return end_attr + 1 - buf; - } - return -1; -} - -struct http_index_cache { - struct http_index_cache *next; - struct url *location; - struct url_list ue; -}; - -static struct http_index_cache *index_cache; - /* * List a directory */ -int -fetchListHTTP(struct url_list *ue, struct url *url, const char *pattern, const char *flags) +struct url_ent * +fetchListHTTP(struct url *url, const char *flags) { - fetchIO *f; - char buf[2 * PATH_MAX]; - size_t buf_len, sum_processed; - ssize_t read_len, processed; - struct index_parser state; - struct http_index_cache *cache = NULL; - int do_cache, ret; - - (void)pattern; - - do_cache = CHECK_FLAG('c'); - - if (do_cache) { - for (cache = index_cache; cache != NULL; cache = cache->next) { - if (strcmp(cache->location->scheme, url->scheme)) - continue; - if (strcmp(cache->location->user, url->user)) - continue; - if (strcmp(cache->location->pwd, url->pwd)) - continue; - if (strcmp(cache->location->host, url->host)) - continue; - if (cache->location->port != url->port) - continue; - if (strcmp(cache->location->doc, url->doc)) - continue; - return fetchAppendURLList(ue, &cache->ue); - } - - cache = malloc(sizeof(*cache)); - if (cache == NULL) - return -1; - fetchInitURLList(&cache->ue); - cache->location = fetchCopyURL(url); - } - - f = fetchGetHTTP(url, flags); - if (f == NULL) { - if (do_cache) { - fetchFreeURLList(&cache->ue); - fetchFreeURL(cache->location); - free(cache); - } - return -1; - } - - state.url = url; - state.state = ST_NONE; - if (do_cache) { - state.ue = &cache->ue; - } else { - state.ue = ue; - } - - buf_len = 0; - - while ((read_len = fetchIO_read(f, buf + buf_len, sizeof(buf) - buf_len)) > 0) { - buf_len += read_len; - sum_processed = 0; - do { - processed = parse_index(&state, buf + sum_processed, buf_len); - if (processed == -1) - break; - buf_len -= processed; - sum_processed += processed; - } while (processed != 0 && buf_len > 0); - if (processed == -1) { - read_len = -1; - break; - } - memmove(buf, buf + sum_processed, buf_len); - } - - fetchIO_close(f); - - ret = read_len < 0 ? -1 : 0; + (void)url; + (void)flags; - if (do_cache) { - if (ret == 0) { - cache->next = index_cache; - index_cache = cache; - } + fprintf(stderr, "fetchListHTTP(): not implemented\n"); + return (NULL); +} - if (fetchAppendURLList(ue, &cache->ue)) - ret = -1; - } +/* + * Arbitrary HTTP verb and content requests + */ +fetchIO * +fetchReqHTTP(struct url *URL, const char *method, const char *flags, + const char *content_type, const char *body) +{ - return ret; + return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags), + flags, content_type, body)); } From b60a033f3ed3e9c853dcf229cfedcdcce4d0ebd5 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 6 Feb 2021 22:53:04 +0100 Subject: [PATCH 02/10] lib/download.c: use freebsd if-modified-since struct field name --- lib/download.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/download.c b/lib/download.c index b3dcc02dc..5dbbf07b5 100644 --- a/lib/download.c +++ b/lib/download.c @@ -150,7 +150,7 @@ xbps_fetch_file_dest_sha256(struct xbps_handle *xhp, const char *uri, const char memset(&st, 0, sizeof(st)); if (stat(filename, &st) == 0) { refetch = true; - url->last_modified = st.st_mtime; + url->ims_time = st.st_mtime; xbps_strlcat(fetch_flags, "i", sizeof(fetch_flags)); } else { if (errno != ENOENT) { @@ -210,8 +210,8 @@ xbps_fetch_file_dest_sha256(struct xbps_handle *xhp, const char *uri, const char xbps_dbg_printf(xhp, "url->doc: %s\n", url->doc); xbps_dbg_printf(xhp, "url->offset: %zd\n", (ssize_t)url->offset); xbps_dbg_printf(xhp, "url->length: %zu\n", url->length); - xbps_dbg_printf(xhp, "url->last_modified: %s\n", - print_time(&url->last_modified)); + xbps_dbg_printf(xhp, "url->ims_time: %s\n", + print_time(&url->ims_time)); /* * If restarting, open the file for appending otherwise create it. */ From cba69e974716fc8167f8fc5ec585b7c0dd796342 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 6 Feb 2021 22:46:50 +0100 Subject: [PATCH 03/10] lib/fetch: fix connection cache --- lib/fetch/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fetch/common.c b/lib/fetch/common.c index 1ee21b175..4ad1fd8a3 100644 --- a/lib/fetch/common.c +++ b/lib/fetch/common.c @@ -823,7 +823,7 @@ fetch_cache_get(const struct url *url, int af) conn_t *conn, *last_conn = NULL; pthread_mutex_lock(&cache_mtx); - for (conn = connection_cache; conn; conn = conn->next) { + for (conn = connection_cache; conn; last_conn = conn, conn = conn->next) { if (conn->port == url->port && strcmp(conn->scheme, url->scheme) == 0 && strcmp(conn->host, url->host) == 0 && From 9353602be2d3c8c284063676f7e38feb04e66e4b Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 6 Feb 2021 23:15:41 +0100 Subject: [PATCH 04/10] lib/fetch: read the full response body before putting http connections into the cache --- lib/fetch/http.c | 80 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 7fca85c5d..647c85e38 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -143,6 +143,7 @@ struct httpio int eof; /* end-of-file flag */ int error; /* error flag */ size_t chunksize; /* remaining size of current chunk */ + off_t contentlength; /* remaining size of the content */ #ifndef NDEBUG size_t total; #endif @@ -222,6 +223,9 @@ http_fillbuf(struct httpio *io, size_t len) if (io->eof) return (0); + if (io->contentlength >= 0 && (off_t)len > io->contentlength) + len = io->contentlength; + /* not chunked: just fetch the requested amount */ if (io->chunked == 0) { if (http_growbuf(io, len) == -1) @@ -231,6 +235,8 @@ http_fillbuf(struct httpio *io, size_t len) return (-1); } io->buflen = nbytes; + if (io->contentlength) + io->contentlength -= io->buflen; io->bufpos = 0; return (io->buflen); } @@ -259,6 +265,8 @@ http_fillbuf(struct httpio *io, size_t len) io->bufpos = 0; io->buflen = nbytes; io->chunksize -= nbytes; + if (io->contentlength >= 0) + io->contentlength -= io->buflen; if (io->chunksize == 0) { char endl[2]; @@ -330,6 +338,16 @@ http_closefn(void *v) if (io->keep_alive) { int val; + for (;;) { + ssize_t rd; + if ((rd = http_fillbuf(io, 512)) == -1) { + r = fetch_close(io->conn); + goto ouch; + } + if (rd == 0) + break; + } + val = 0; setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)); @@ -344,16 +362,26 @@ http_closefn(void *v) r = fetch_close(io->conn); } +ouch: free(io->buf); free(io); return (r); } +static void +http_funinit(struct httpio *io, conn_t *conn, int chunked, int keep_alive, off_t clength) +{ + io->conn = conn; + io->chunked = chunked; + io->contentlength = clength; + io->keep_alive = keep_alive; +} + /* * Wrap a file descriptor up */ static fetchIO * -http_funopen(conn_t *conn, int chunked, int keep_alive) +http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength) { struct httpio *io; fetchIO *f; @@ -362,9 +390,7 @@ http_funopen(conn_t *conn, int chunked, int keep_alive) fetch_syserr(); return (NULL); } - io->conn = conn; - io->chunked = chunked; - io->keep_alive = keep_alive; + http_funinit(io, conn, chunked, keep_alive, clength); f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn); if (f == NULL) { fetch_syserr(); @@ -1537,6 +1563,21 @@ http_request(struct url *URL, const char *op, struct url_stat *us, return (http_request_body(URL, op, us, purl, flags, NULL, NULL)); } +static int +slurp_body(conn_t *conn, int chunked, int keep_alive, size_t clength) +{ + struct httpio io = {0}; + http_funinit(&io, conn, chunked, keep_alive, clength); + for (;;) { + ssize_t rd; + if ((rd = http_fillbuf(&io, 512)) == -1) + return -1; + if (rd == 0) + break; + } + return 0; +} + /* * Send a request and process the reply * @@ -1960,7 +2001,14 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, /* all other cases: we got a redirect */ e = conn->err; clean_http_auth_challenges(&server_challenges); - fetch_close(conn); + if (keep_alive) { + if (slurp_body(conn, chunked, keep_alive, clength) == -1) + fetch_close(conn); + else + fetch_cache_put(conn, fetch_close); + } else { + fetch_close(conn); + } conn = NULL; if (!new) { DEBUGF("redirect with no new location\n"); @@ -1985,10 +2033,9 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, http_seterr(HTTP_NOT_MODIFIED); if (keep_alive) { fetch_cache_put(conn, fetch_close); - } else { - fetch_close(conn); + conn = NULL; } - return (NULL); + goto ouch; } /* check for inconsistencies */ @@ -2023,8 +2070,19 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, URL->offset = offset; URL->length = clength; + if (HTTP_ERROR(conn->err)) { + if (keep_alive) { + if (slurp_body(conn, chunked, keep_alive, clength) == -1) { + fetch_close(conn); + } + fetch_cache_put(conn, fetch_close); + conn = NULL; + } + goto ouch; + } + /* wrap it up in a fetchIO */ - if ((f = http_funopen(conn, chunked, keep_alive)) == NULL) { + if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) { fetch_syserr(); goto ouch; } @@ -2034,10 +2092,6 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, if (purl) fetchFreeURL(purl); - if (HTTP_ERROR(conn->err)) { - fetchIO_close(f); - f = NULL; - } clean_http_headerbuf(&headerbuf); clean_http_auth_challenges(&server_challenges); clean_http_auth_challenges(&proxy_challenges); From 16af9567f432311157fc96c7703128ffba4d79ad Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 6 Feb 2021 23:34:30 +0100 Subject: [PATCH 05/10] lib/fetch: remove unused cached variable --- lib/fetch/http.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 647c85e38..6bc74695b 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1432,7 +1432,7 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs, * Connect to the correct HTTP server or proxy. */ static conn_t * -http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) +http_connect(struct url *URL, struct url *purl, const char *flags) { struct url *curl; conn_t *conn; @@ -1462,10 +1462,8 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) curl = (purl != NULL) ? purl : URL; - if ((conn = fetch_cache_get(curl, af)) != NULL) { - *cached = 1; + if ((conn = fetch_cache_get(curl, af)) != NULL) return (conn); - } if ((conn = fetch_connect(curl, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ @@ -1593,7 +1591,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, char hbuf[URL_HOSTLEN + 7], *host; conn_t *conn; struct url *url, *new; - int chunked, direct, ims, keep_alive, noredirect, verbose, cached; + int chunked, direct, ims, keep_alive, noredirect, verbose; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; @@ -1637,7 +1635,6 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, length = -1; size = -1; mtime = 0; - cached = 0; /* check port */ if (!url->port) @@ -1652,7 +1649,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, } /* connect to server or proxy */ - if ((conn = http_connect(url, purl, flags, &cached)) == NULL) + if ((conn = http_connect(url, purl, flags)) == NULL) goto ouch; /* append port number only if necessary */ @@ -1855,11 +1852,6 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, case HTTP_PROTOCOL_ERROR: /* fall through */ case -1: -#if 0 - --i; - if (cached) - continue; -#endif fetch_syserr(); goto ouch; default: From e76fdbad6f0ad9f6001060bdaf749065be84f577 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 7 Feb 2021 00:45:25 +0100 Subject: [PATCH 06/10] lib/fetch: default to keep-alive with HTTP/1.1 --- lib/fetch/http.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 6bc74695b..51cdd0db2 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -474,7 +474,7 @@ http_cmd(conn_t *conn, const char *fmt, ...) * Get and parse status line */ static int -http_get_reply(conn_t *conn) +http_get_reply(conn_t *conn, int *keep_alive) { char *p; @@ -493,7 +493,12 @@ http_get_reply(conn_t *conn) return (HTTP_PROTOCOL_ERROR); p = conn->buf + 4; if (*p == '/') { - if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1')) + if (p[1] != '1' || p[2] != '.') + return (HTTP_PROTOCOL_ERROR); + if (p[3] == '1') { + if (keep_alive) + *keep_alive = 1; + } else if (p[3] != '0') return (HTTP_PROTOCOL_ERROR); p += 4; } @@ -1475,7 +1480,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags) http_cmd(conn, "Host: %s:%d", URL->host, URL->port); http_cmd(conn, "%s", ""); - if (http_get_reply(conn) != HTTP_OK) { + if (http_get_reply(conn, NULL) != HTTP_OK) { http_seterr(conn->err); goto ouch; } @@ -1799,7 +1804,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, sizeof(val)); /* get reply */ - switch (http_get_reply(conn)) { + switch (http_get_reply(conn, &keep_alive)) { case HTTP_OK: case HTTP_PARTIAL: case HTTP_NOT_MODIFIED: @@ -1875,6 +1880,10 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; case hdr_connection: + if (keep_alive) { + keep_alive = (strcasecmp(p, "close") != 0); + break; + } /* XXX too weak? */ keep_alive = (strcasecmp(p, "keep-alive") == 0); break; From 5399443de636e5f8e4fb64001d3e848dcdc3d3b2 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 7 Feb 2021 01:09:11 +0100 Subject: [PATCH 07/10] lib/fetch: read the trailer of the end chunk so keep-alive with chunked encoding works --- lib/fetch/http.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 51cdd0db2..4925f9c44 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -249,7 +249,8 @@ http_fillbuf(struct httpio *io, size_t len) return (-1); case 0: io->eof = 1; - return (0); + io->buflen = 0; + goto trailer; } } @@ -268,6 +269,7 @@ http_fillbuf(struct httpio *io, size_t len) if (io->contentlength >= 0) io->contentlength -= io->buflen; +trailer: if (io->chunksize == 0) { char endl[2]; ssize_t len2; From f0058717267ed6d2694ec3fbb4fe84bb7bdc8fdb Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 7 Feb 2021 01:32:36 +0100 Subject: [PATCH 08/10] lib/fetch: switch to gmtime_r --- lib/fetch/http.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 4925f9c44..4073cd011 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1596,6 +1596,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, { char timebuf[80]; char hbuf[URL_HOSTLEN + 7], *host; + struct tm timestruct; conn_t *conn; struct url *url, *new; int chunked, direct, ims, keep_alive, noredirect, verbose; @@ -1605,7 +1606,6 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, const char *p; fetchIO *f; hdr_t h; - struct tm *timestruct; http_headerbuf_t headerbuf; http_auth_challenges_t server_challenges; http_auth_challenges_t proxy_challenges; @@ -1679,9 +1679,9 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us, } if (ims && url->ims_time) { - timestruct = gmtime((time_t *)&url->ims_time); + gmtime_r(&url->ims_time, ×truct); (void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT", - timestruct); + ×truct); if (verbose) fetch_info("If-Modified-Since: %s", timebuf); http_cmd(conn, "If-Modified-Since: %s", timebuf); From e404e839618bc60b277ecb3d97288731b3165d75 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 7 Feb 2021 01:43:16 +0100 Subject: [PATCH 09/10] lib/fetch: pass correctly sized array just in case, found by lgtm.com --- lib/fetch/http.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 4073cd011..faa3a6239 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1315,6 +1315,7 @@ http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c, { HASHHEX HA1; HASHHEX digest; + HASHHEX empty = {0}; int r; char noncecount[10]; char cnonce[40]; @@ -1347,7 +1348,7 @@ http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c, parms->password, c->nonce, cnonce, HA1); DEBUGF("HA1: [%s]\n", HA1); DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop, - "GET", url->doc, "", digest); + "GET", url->doc, empty, digest); if (c->qop[0]) { r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\"," From 551daa91a529fc196fc89a86d782334ddceb29ef Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 7 Feb 2021 13:10:00 +0100 Subject: [PATCH 10/10] lib/fetch: fix memory leak when discarding redirect and error reply bodys --- lib/fetch/http.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index faa3a6239..afc255a72 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1581,6 +1581,7 @@ slurp_body(conn_t *conn, int chunked, int keep_alive, size_t clength) if (rd == 0) break; } + free(io.buf); return 0; }