Skip to content

Check libcurl return value #138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Check libcurl return value #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3e98332
Check libcurl return value
dentiny Sep 27, 2025
574039f
fix header inclusion
dentiny Sep 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 49 additions & 51 deletions src/httpfs_curl_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
#include "httpfs_curl_client.hpp"
#endif

#define CHECK_CURL_OK(expr) D_ASSERT((expr) == CURLE_OK)

namespace duckdb {

// we statically compile in libcurl, which means the cert file location of the build machine is the
Expand Down Expand Up @@ -91,11 +93,11 @@ CURLHandle::CURLHandle(const string &token, const string &cert_path) {
throw InternalException("Failed to initialize curl");
}
if (!token.empty()) {
curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str());
curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER);
CHECK_CURL_OK(curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()));
CHECK_CURL_OK(curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER));
}
if (!cert_path.empty()) {
curl_easy_setopt(curl, CURLOPT_CAINFO, cert_path.c_str());
CHECK_CURL_OK(curl_easy_setopt(curl, CURLOPT_CAINFO, cert_path.c_str()));
}
}

Expand All @@ -110,7 +112,7 @@ struct RequestInfo {
std::vector<HTTPHeaders> header_collection;
};

static idx_t httpfs_client_count = 0;
static std::atomic<idx_t> httpfs_client_count {0};

class HTTPFSCurlClient : public HTTPClient {
public:
Expand All @@ -129,45 +131,48 @@ class HTTPFSCurlClient : public HTTPClient {

// set curl options
// follow redirects
curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L));

// Curl re-uses connections by default
if (!http_params.keep_alive) {
curl_easy_setopt(*curl, CURLOPT_FORBID_REUSE, 1L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_FORBID_REUSE, 1L));
}

if (http_params.enable_curl_server_cert_verification) {
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 1L); // Verify the cert
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, 2L); // Verify that the cert matches the hostname
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 1L)); // Verify the cert
CHECK_CURL_OK(
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, 2L)); // Verify that the cert matches the hostname
} else {
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 0L); // Override default, don't verify the cert
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST,
0L); // Override default, don't verify that the cert matches the hostname
CHECK_CURL_OK(
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 0L)); // Override default, don't verify the cert
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST,
0L)); // Override default, don't verify that the cert matches the hostname
}

// set read timeout
curl_easy_setopt(*curl, CURLOPT_TIMEOUT, http_params.timeout);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_TIMEOUT, http_params.timeout));
// set connection timeout
curl_easy_setopt(*curl, CURLOPT_CONNECTTIMEOUT, http_params.timeout);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_CONNECTTIMEOUT, http_params.timeout));
// accept content as-is (i.e no decompressing)
curl_easy_setopt(*curl, CURLOPT_ACCEPT_ENCODING, "identity");
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_ACCEPT_ENCODING, "identity"));
// follow redirects
curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L));

// define the header callback
curl_easy_setopt(*curl, CURLOPT_HEADERFUNCTION, RequestHeaderCallback);
curl_easy_setopt(*curl, CURLOPT_HEADERDATA, &request_info->header_collection);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HEADERFUNCTION, RequestHeaderCallback));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HEADERDATA, &request_info->header_collection));
// define the write data callback (for get requests)
curl_easy_setopt(*curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback);
curl_easy_setopt(*curl, CURLOPT_WRITEDATA, &request_info->body);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_WRITEDATA, &request_info->body));

if (!http_params.http_proxy.empty()) {
curl_easy_setopt(*curl, CURLOPT_PROXY,
StringUtil::Format("%s:%s", http_params.http_proxy, http_params.http_proxy_port).c_str());
CHECK_CURL_OK(curl_easy_setopt(
*curl, CURLOPT_PROXY,
StringUtil::Format("%s:%s", http_params.http_proxy, http_params.http_proxy_port).c_str()));

if (!http_params.http_proxy_username.empty()) {
curl_easy_setopt(*curl, CURLOPT_PROXYUSERNAME, http_params.http_proxy_username.c_str());
curl_easy_setopt(*curl, CURLOPT_PROXYPASSWORD, http_params.http_proxy_password.c_str());
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_PROXYUSERNAME, http_params.http_proxy_username.c_str()));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_PROXYPASSWORD, http_params.http_proxy_password.c_str()));
}
}
}
Expand All @@ -191,9 +196,9 @@ class HTTPFSCurlClient : public HTTPClient {
CURLcode res;
{
// If the same handle served a HEAD request, we must set NOBODY back to 0L to request content again
curl_easy_setopt(*curl, CURLOPT_NOBODY, 0L);
curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_NOBODY, 0L));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str()));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr));
res = curl->Execute();
}

Expand Down Expand Up @@ -237,15 +242,15 @@ class HTTPFSCurlClient : public HTTPClient {

CURLcode res;
{
curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str()));
// Perform PUT
curl_easy_setopt(*curl, CURLOPT_CUSTOMREQUEST, "PUT");
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_CUSTOMREQUEST, "PUT"));
// Include PUT body
curl_easy_setopt(*curl, CURLOPT_POSTFIELDS, const_char_ptr_cast(info.buffer_in));
curl_easy_setopt(*curl, CURLOPT_POSTFIELDSIZE, info.buffer_in_len);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_POSTFIELDS, const_char_ptr_cast(info.buffer_in)));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_POSTFIELDSIZE, info.buffer_in_len));

// Apply headers
curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr));

res = curl->Execute();
}
Expand All @@ -271,14 +276,14 @@ class HTTPFSCurlClient : public HTTPClient {
CURLcode res;
{
// Set URL
curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str()));

// Perform HEAD request instead of GET
curl_easy_setopt(*curl, CURLOPT_NOBODY, 1L);
curl_easy_setopt(*curl, CURLOPT_HTTPGET, 0L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_NOBODY, 1L));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPGET, 0L));

// Add headers if any
curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr));

// Execute HEAD request
res = curl->Execute();
Expand All @@ -304,16 +309,16 @@ class HTTPFSCurlClient : public HTTPClient {
CURLcode res;
{
// Set URL
curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str()));

// Set DELETE request method
curl_easy_setopt(*curl, CURLOPT_CUSTOMREQUEST, "DELETE");
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_CUSTOMREQUEST, "DELETE"));

// Follow redirects
curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_FOLLOWLOCATION, 1L));

// Add headers if any
curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr));

// Execute DELETE request
res = curl->Execute();
Expand Down Expand Up @@ -342,15 +347,15 @@ class HTTPFSCurlClient : public HTTPClient {

CURLcode res;
{
curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
curl_easy_setopt(*curl, CURLOPT_POST, 1L);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str()));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_POST, 1L));

// Set POST body
curl_easy_setopt(*curl, CURLOPT_POSTFIELDS, const_char_ptr_cast(info.buffer_in));
curl_easy_setopt(*curl, CURLOPT_POSTFIELDSIZE, info.buffer_in_len);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_POSTFIELDS, const_char_ptr_cast(info.buffer_in)));
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_POSTFIELDSIZE, info.buffer_in_len));

// Add headers if any
curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr);
CHECK_CURL_OK(curl_easy_setopt(*curl, CURLOPT_HTTPHEADER, curl_headers ? curl_headers.headers : nullptr));

// Execute POST request
res = curl->Execute();
Expand Down Expand Up @@ -430,13 +435,7 @@ class HTTPFSCurlClient : public HTTPClient {
optional_ptr<HTTPState> state;
unique_ptr<RequestInfo> request_info;

static std::mutex &GetRefLock() {
static std::mutex mtx;
return mtx;
}

static void InitCurlGlobal() {
GetRefLock();
if (httpfs_client_count == 0) {
curl_global_init(CURL_GLOBAL_DEFAULT);
}
Expand All @@ -446,7 +445,6 @@ class HTTPFSCurlClient : public HTTPClient {
static void DestroyCurlGlobal() {
// TODO: when to call curl_global_cleanup()
// calling it on client destruction causes SSL errors when verification is on (due to many requests).
// GetRefLock();
// if (httpfs_client_count == 0) {
// throw InternalException("Destroying Httpfs client that did not initialize CURL");
// }
Expand Down
2 changes: 1 addition & 1 deletion src/include/httpfs_curl_client.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

#include "duckdb/common/http_util.hpp"

#include <curl/curl.h>
namespace duckdb {
class HTTPLogger;
class FileOpener;
struct FileOpenerInfo;
class HTTPState;

#include <curl/curl.h>
class CURLHandle {
public:
CURLHandle(const string &token, const string &cert_path);
Expand Down
2 changes: 1 addition & 1 deletion src/s3fs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
if (use_sse_kms) {
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
}
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
if (content_type.length() > 0) {
canonical_request += "\ncontent-type:" + content_type;
}
Expand Down
Loading