From 55f2a67574b800fabf6c89d2b6465141b761dae5 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 21 Feb 2024 05:04:29 +0100 Subject: [PATCH 01/13] rbreaking change: prefer IPV6 This change implements the happy eyeballs strategy to bnring better support of IPv6 for hackney. It follows the recommendations iof the RFC 8305. https://datatracker.ietf.org/doc/html/rfc8305 --- NOTICE | 3 + rebar.lock | 3 + src/hackney_connection.erl | 20 +- src/hackney_happy.erl | 72 +++++++ src/hackney_http_connect.erl | 2 +- src/hackney_pool.erl | 8 +- src/hackney_socks5.erl | 2 +- src/hackney_ssl.erl | 30 ++- src/hackney_tcp.erl | 2 +- src/libs/hackney_cidr.erl | 243 ++++++++++++++++++++++ test/hackney_cidr_tests.erl | 377 +++++++++++++++++++++++++++++++++++ 11 files changed, 720 insertions(+), 42 deletions(-) create mode 100644 src/hackney_happy.erl create mode 100644 src/libs/hackney_cidr.erl create mode 100644 test/hackney_cidr_tests.erl diff --git a/NOTICE b/NOTICE index be905996..ef394ed6 100644 --- a/NOTICE +++ b/NOTICE @@ -21,3 +21,6 @@ Copyright (c) 2009, Erlang Training and Consulting Ltd. Copyright (C) 1998 - 2014, Daniel Stenberg, , et al. *) hackney_trace (C) 2015 under the Erlang Public LicensE + +*) hackney_cidr is based on inet_cidr 1.2.1. vendored for customer purpose. +Copyright (c) 2024, Enki Multimedia , MIT License diff --git a/rebar.lock b/rebar.lock index ee72ef3e..29b16ba8 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,6 +1,7 @@ {"1.2.0", [{<<"certifi">>,{pkg,<<"certifi">>,<<"2.12.0">>},0}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},0}, + {<<"inet_cidr">>,{pkg,<<"erl_cidr">>,<<"1.2.1">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},0}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},0}, @@ -10,6 +11,7 @@ {pkg_hash,[ {<<"certifi">>, <<"2D1CCA2EC95F59643862AF91F001478C9863C2AC9CB6E2F89780BFD8DE987329">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, + {<<"inet_cidr">>, <<"921BB463BCF10EAD937AE491E5BCAC2823E550B339A9893AC6574518553CC067">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, @@ -18,6 +20,7 @@ {pkg_hash_ext,[ {<<"certifi">>, <<"EE68D85DF22E554040CDB4BE100F33873AC6051387BAF6A8F6CE82272340FF1C">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, + {<<"inet_cidr">>, <<"FCF5D51B1CC1B26D1748AB39F9B614AB445AD17172A9396D48C6B5C15EA9EFCF">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, diff --git a/src/hackney_connection.erl b/src/hackney_connection.erl index 3dc95a60..ceac66c4 100644 --- a/src/hackney_connection.erl +++ b/src/hackney_connection.erl @@ -102,27 +102,13 @@ connect_options(hackney_local_tcp, _Host, ClientOptions) -> proplists:get_value(connect_options, ClientOptions, []); connect_options(Transport, Host, ClientOptions) -> - ConnectOpts0 = proplists:get_value(connect_options, ClientOptions, []), - - %% handle ipv6 - ConnectOpts1 = case lists:member(inet, ConnectOpts0) orelse - lists:member(inet6, ConnectOpts0) of - true -> - ConnectOpts0; - false -> - case hackney_util:is_ipv6(Host) of - true -> - [inet6 | ConnectOpts0]; - false -> - ConnectOpts0 - end - end, + ConnectOpts = proplists:get_value(connect_options, ClientOptions, []), case Transport of hackney_ssl -> - ConnectOpts1 ++ ssl_opts(Host, ClientOptions); + [{ssl_options, ssl_opts(Host, ClientOptions)} | ConnectOpts]; _ -> - ConnectOpts1 + ConnectOpts end. diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl new file mode 100644 index 00000000..0a90dc01 --- /dev/null +++ b/src/hackney_happy.erl @@ -0,0 +1,72 @@ +-module(hackney_happy). + +-export([connect/3, connect/4]). + +-include("hackney_internal.hrl"). +-include_lib("kernel/include/inet.hrl"). + +-define(TIMEOUT, 250). +-define(CONNECT_TIMEOUT, 5000). + +connect(Hostname, Port, Opts) -> + connect(Hostname, Port, Opts, ?CONNECT_TIMEOUT). + +connect(Hostname, Port, Opts, Timeout) -> + case hackney_cidr:is_ipv6(Hostname) of + true -> + ?report_debug("connect using IPv6", [{hostname, Hostname}, {port, Port}]), + gen_tcp:connect(Hostname, Port, [inet6 | Opts], Timeout); + false -> + case hackney_cidr:is_ipv4(Hostname) of + true -> + ?report_debug("connect using IPv4", [{hostname, Hostname}, {port, Port}]), + gen_tcp:connect(Hostname, Port, [inet | Opts], Timeout); + false -> + ?report_debug("happy eyeballs, try to connect using IPv6", [{hostname, Hostname}, {port, Port}]), + Self = self(), + Addrs = getaddrs(Hostname), + Pid = spawn_link( fun() -> try_connect(Addrs, Port, Opts, Self, {error, nxdomain}) end), + MRef = erlang:monitor(process, Pid), + receive + {happy_connect, OK} -> + erlang:demonitor(MRef, [flush]), + OK; + {'DOWN', MRef, _Type, _Pid, Info} -> + {'error', {'connect_error', Info}} + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error, connect_timeout} + end + end + end. + +getaddrs(Hostname) -> + IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Hostname, 'aaaa')], + IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Hostname, 'a')], + IP6Addrs ++ IP4Addrs. + +getbyname(Hostname, Type) -> + case (catch inet_res:getbyname(Hostname, Type)) of + {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); + {error, _Reason} -> []; + Else -> + %% ERLANG 22 has an issue when g matching somee DNS server messages + ?report_debug("DNS error", [{hostname, Hostname} + ,{type, Type} + ,{error, Else}]), + [] + end. + +try_connect([], _Port, _Opts, ServerPid, LastError) -> + ?report_trace("happy eyeball: failed to connect", [{error, LastError}]), + ServerPid ! {hackney_happy, LastError}; +try_connect([{IP, Type} | Rest], Port, Opts, ServerPid, _LastError) -> + ?report_trace("try to connect", [{ip, IP}, {type, Type}]), + case gen_tcp:connect(IP, Port, [Type | Opts], ?TIMEOUT) of + {ok, Socket} = OK -> + ?report_trace("success to connect", [{ip, IP}, {type, Type}]), + ok = gen_tcp:controlling_process(Socket, ServerPid), + ServerPid ! {happy_connect, OK}; + Error -> + try_connect(Rest, Port, Opts, ServerPid, Error) + end. diff --git a/src/hackney_http_connect.erl b/src/hackney_http_connect.erl index 60861cf5..03299313 100644 --- a/src/hackney_http_connect.erl +++ b/src/hackney_http_connect.erl @@ -63,7 +63,7 @@ connect(ProxyHost, ProxyPort, Opts, Timeout) ConnectOpts = hackney_util:filter_options(Opts, AcceptedOpts, BaseOpts), %% connect to the proxy, and upgrade the socket if needed. - case gen_tcp:connect(ProxyHost, ProxyPort, ConnectOpts) of + case hackney_happy:connect(ProxyHost, ProxyPort, ConnectOpts) of {ok, Socket} -> case do_handshake(Socket, Host, Port, Opts) of ok -> diff --git a/src/hackney_pool.erl b/src/hackney_pool.erl index b51feaa6..852ad867 100644 --- a/src/hackney_pool.erl +++ b/src/hackney_pool.erl @@ -63,12 +63,13 @@ checkout(Host, Port, Transport, Client) -> Requester = self(), try do_checkout(Requester, Host, Port, Transport, Client) - catch _:_ -> + catch _:Error -> + ?report_trace("pool: checkout failure", [{error, Error}]), {error, checkout_failure} end. do_checkout(Requester, Host, _Port, Transport, #client{options=Opts, - mod_metrics=Metrics}=Client) -> + mod_metrics=Metrics}=Client) -> ConnectTimeout = proplists:get_value(connect_timeout, Opts, 8000), %% Fall back to using connect_timeout if checkout_timeout is not set CheckoutTimeout = proplists:get_value(checkout_timeout, Opts, ConnectTimeout), @@ -78,7 +79,6 @@ do_checkout(Requester, Host, _Port, Transport, #client{options=Opts, Pool = find_pool(PoolName, Opts), case catch gen_server:call(Pool, {checkout, Connection, Requester, RequestRef}, CheckoutTimeout) of {ok, Socket, Owner} -> - %% stats ?report_debug("reuse a connection", [{pool, PoolName}]), _ = metrics:update_meter(Metrics, [hackney_pool, PoolName, take_rate], 1), @@ -105,7 +105,7 @@ do_checkout(Requester, Host, _Port, Transport, #client{options=Opts, _ = metrics:increment_counter(Metrics, [hackney, Host, connect_timeout]), {error, timeout}; Error -> - ?report_trace("connect error", []), + ?report_trace("connect error", [{pool, PoolName}, {error, Error}]), _ = metrics:increment_counter(Metrics, [hackney, Host, connect_error]), Error end; diff --git a/src/hackney_socks5.erl b/src/hackney_socks5.erl index 4be6e77d..c7dcac26 100644 --- a/src/hackney_socks5.erl +++ b/src/hackney_socks5.erl @@ -62,7 +62,7 @@ connect(Host, Port, Opts, Timeout) when is_list(Host), is_integer(Port), ConnectOpts = hackney_util:filter_options(Opts, AcceptedOpts, BaseOpts), %% connect to the socks 5 proxy - case gen_tcp:connect(ProxyHost, ProxyPort, ConnectOpts, Timeout) of + case hackney_happy:connect(ProxyHost, ProxyPort, ConnectOpts, Timeout) of {ok, Socket} -> case do_handshake(Socket, Host, Port, Opts) of ok -> diff --git a/src/hackney_ssl.erl b/src/hackney_ssl.erl index 5b65e4a6..79e1015b 100644 --- a/src/hackney_ssl.erl +++ b/src/hackney_ssl.erl @@ -25,16 +25,7 @@ %% @doc Atoms used to identify messages in {active, once | true} mode. messages(_) -> {ssl, ssl_closed, ssl_error}. -%% @doc The ssl:connect/4 (and related) doesn't work with textual representation -%% of IP addresses. It accepts either a string with a DNS-resolvable name or a -%% tuple with parts of an IP as numbers. This function attempts to parse given -%% string and either returns such tuple, or the string if it's impossible to -%% parse. -parse_address(Host) when is_list(Host) -> - case inet:parse_address(Host) of - {ok, Address} -> Address; - {error, _} -> Host - end. + check_hostname_opts(Host0) -> @@ -134,14 +125,17 @@ find(_Fun, []) -> connect(Host, Port, Opts) -> connect(Host, Port, Opts, infinity). -connect(Host, Port, Opts, Timeout) when is_list(Host), is_integer(Port), - (Timeout =:= infinity orelse is_integer(Timeout)) -> - BaseOpts = [binary, {active, false}, {packet, raw}, - {secure_renegotiate, true}, - {reuse_sessions, true}], - Opts1 = hackney_util:merge_opts(BaseOpts, Opts), - %% connect - ssl:connect(parse_address(Host), Port, Opts1, Timeout). +connect(Host, Port, Opts0, Timeout) when is_list(Host), is_integer(Port), + (Timeout =:= 5000 orelse is_integer(Timeout)) -> + SSLOpts = [{secure_renegotiate, true}, proplists:get_value(ssl_options, Opts0)], + BaseOpts = [binary, {active, false}, {packet, raw}], + Opts1 = hackney_util:merge_opts(BaseOpts, proplists:delete(ssl_options, Opts0)), + case hackney_happy:connect(Host, Port, Opts1, Timeout) of + {ok, Sock} -> + ssl:connect(Sock, SSLOpts); + Error -> + Error + end. diff --git a/src/hackney_tcp.erl b/src/hackney_tcp.erl index 506d9c37..cb21cd9f 100644 --- a/src/hackney_tcp.erl +++ b/src/hackney_tcp.erl @@ -28,7 +28,7 @@ connect(Host, Port, Opts, Timeout) when is_list(Host), is_integer(Port), BaseOpts = [binary, {active, false}, {packet, raw}], Opts1 = hackney_util:merge_opts(BaseOpts, Opts), %% connect - gen_tcp:connect(Host, Port, Opts1, Timeout). + hackney_happy:connect(Host, Port, Opts1, Timeout). recv(Socket, Length) -> recv(Socket, Length, infinity). diff --git a/src/libs/hackney_cidr.erl b/src/libs/hackney_cidr.erl new file mode 100644 index 00000000..b308f79d --- /dev/null +++ b/src/libs/hackney_cidr.erl @@ -0,0 +1,243 @@ +%%% -*- erlang -*- +%%% This file is part of inet_cidr eleased under the MIT license. +%%% See the NOTICE for more information. +%%% +%%% Copyright (c) 2016-2024 Benoît Chesneau + +-module(hackney_cidr). + +-export([parse/1, parse/2]). +-export([address_count/2]). +-export([contains/2]). +-export([usort_cidrs/1]). +-export([merge_cidrs/1]). +-export([to_string/1]). +-export([to_binary/1]). +-export([is_ipv4/1]). +-export([is_ipv6/1]). + +-type cidr() :: {Start :: inet:ip4_address(), End :: inet:ip4_address(), MaskLen :: 0..32} + | {Start :: inet:ip6_address(), End :: inet:ip6_address(), MaskLen :: 0..128}. + +-export_type([cidr/0]). + +-spec parse(string() | binary()) -> cidr(). +%% @doc parses S as a CIDR notation IP address and mask +parse(S) -> + parse(S, false). + +-spec parse(string() | binary(), Adjust :: boolean()) -> cidr(). +%% @doc parses S as a CIDR notation IP address and mask. +%% If Adjust = `true', allow the IP to contain values beyond the mask and +%% silently ignore them. Otherwise, enforce that the IP address is fully inside +%% the specified mask (the default behavior of `parse/1'). +parse(B, Adjust) when is_binary(B) -> + parse(binary_to_list(B), Adjust); +parse(S, Adjust) -> + {StartAddr, PrefixLen} = parse_cidr(S, Adjust), + EndAddr = calc_end_address(StartAddr, PrefixLen), + {StartAddr, EndAddr, PrefixLen}. + +-spec address_count(inet:ip4_address(), MaskLen :: 0..32) -> pos_integer(); + (inet:ip6_address(), MaskLen :: 0..128) -> pos_integer(). +%% @doc return the number of IP addresses included in the CIDR block +address_count(IP, Len) -> + 1 bsl (bit_count(IP) - Len). + +-spec contains(cidr(), inet:ip_address() | cidr()) -> boolean(). +%% @doc return true if the CIDR block contains the IP address or CIDR block, false otherwise. +contains({StartAddr, EndAddr, _L}, Addr) when tuple_size(StartAddr) == tuple_size(EndAddr), + tuple_size(StartAddr) == tuple_size(Addr) -> + ip_gte(Addr, StartAddr) andalso ip_lte(Addr, EndAddr); + +contains({StartAddr1, EndAddr1, _L1}, + {StartAddr2, EndAddr2, _L2}) when tuple_size(StartAddr1) == tuple_size(EndAddr1), + tuple_size(EndAddr1) == tuple_size(StartAddr2), + tuple_size(StartAddr2) == tuple_size(EndAddr2) -> + ip_gte(StartAddr2, StartAddr1) andalso ip_lte(StartAddr2, EndAddr1) andalso + ip_gte(EndAddr2, StartAddr1) andalso ip_lte(EndAddr2, EndAddr1); + +contains(_, _) -> + false. + +-spec usort_cidrs([cidr()]) -> [cidr()]. +%% @doc Unique sort a list of CIDR blocks, ordering IPv4 ranges before IPv6 ranges +usort_cidrs(CIDRs) -> + lists:usort(fun cidr_lte/2, CIDRs). + +-spec merge_cidrs([cidr()]) -> [cidr()]. +%% @doc Unique sort and merge a list of CIDR blocks, ordering IPv4 ranges before IPv6 ranges. +%% For merging, CIDR blocks that are contained by other CIDR blocks are removed and +%% adjacent CIDR blocks are merged into larger ones. +merge_cidrs(CIDRs) -> + merge_sorted_cidrs(usort_cidrs(CIDRs)). + +-spec to_string(cidr()) -> string(). +%% @doc return a CIDR block as a string. +to_string({StartAddr, _EndAddr, Len}) -> + inet:ntoa(StartAddr) ++ "/" ++ integer_to_list(Len). + +-spec to_binary(cidr()) -> binary(). +%% @doc return a CIDR block as a binary string. +to_binary({StartAddr, _EndAddr, Len}) -> + <<(list_to_binary(inet:ntoa(StartAddr)))/binary, "/", (integer_to_binary(Len))/binary>>. + +-spec is_ipv4(inet:ip_address()) -> boolean(). +%% @doc return true if the value is an ipv4 address +is_ipv4({A, B, C, D}) -> + (((A >= 0) andalso (A =< 255)) andalso + ((B >= 0) andalso (B =< 255)) andalso + ((C >= 0) andalso (C =< 255)) andalso + ((D >= 0) andalso (D =< 255))); +is_ipv4(_) -> + false. + +-spec is_ipv6(inet:ip_address()) -> boolean(). +%% @doc return true if the value is an ipv6 address +is_ipv6({A, B, C, D, E, F, G, H}) -> + (((A >= 0) andalso (A =< 65535)) andalso + ((B >= 0) andalso (B =< 65535)) andalso + ((C >= 0) andalso (C =< 65535)) andalso + ((D >= 0) andalso (D =< 65535)) andalso + ((E >= 0) andalso (E =< 65535)) andalso + ((F >= 0) andalso (F =< 65535)) andalso + ((G >= 0) andalso (G =< 65535)) andalso + ((H >= 0) andalso (H =< 65535))); +is_ipv6(_) -> + false. + +%% internals + +bit_count({_, _, _, _}) -> 32; +bit_count({_, _, _, _, _, _, _, _}) -> 128. + +parse_cidr(S, Adjust) -> + {StartAddr, Masked, PrefixLen} = + case re:split(S, "/", [{return, list}, {parts, 2}]) of + [Prefix, LenStr] -> + {ok, Addr} = inet:parse_address(Prefix), + {PLen, _} = string:to_integer(LenStr), + {Addr, band_with_mask(Addr, start_mask(Addr, PLen)), PLen}; + [Prefix] -> + {ok, Addr} = inet:parse_address(Prefix), + PLen = case is_ipv6(Addr) of + true -> 128; + false -> 32 + end, + {Addr, band_with_mask(Addr, start_mask(Addr, PLen)), PLen} + end, + if + Adjust /= true, Masked /= StartAddr -> error(invalid_cidr); + true -> ok + end, + {Masked, PrefixLen}. + +start_mask({_, _, _, _}=Addr, Len) when Len >= 0, Len =< 32 -> + {A, B, C, D} = end_mask(Addr, Len), + {bnot A, bnot B, bnot C, bnot D}; + +start_mask({_, _, _, _, _, _, _, _}=Addr, Len) when Len >= 0, Len =< 128 -> + {A, B, C, D, E, F, G, H} = end_mask(Addr, Len), + {bnot A, bnot B, bnot C, bnot D, bnot E, bnot F, bnot G, bnot H}. + +end_mask({_, _, _, _}, Len) when Len >= 0, Len =< 32 -> + if + Len == 32 -> {0, 0, 0, 0}; + Len >= 24 -> {0, 0, 0, bmask(Len, 8)}; + Len >= 16 -> {0, 0, bmask(Len, 8), 16#FF}; + Len >= 8 -> {0, bmask(Len, 8), 16#FF, 16#FF}; + Len >= 0 -> {bmask(Len, 8), 16#FF, 16#FF, 16#FF} + end; + +end_mask({_, _, _, _, _, _, _, _}, Len) when Len >= 0, Len =< 128 -> + if + Len == 128 -> {0, 0, 0, 0, 0, 0, 0, 0}; + Len >= 112 -> {0, 0, 0, 0, 0, 0, 0, bmask(Len, 16)}; + Len >= 96 -> {0, 0, 0, 0, 0, 0, bmask(Len, 16), 16#FFFF}; + Len >= 80 -> {0, 0, 0, 0, 0, bmask(Len, 16), 16#FFFF, 16#FFFF}; + Len >= 64 -> {0, 0, 0, 0, bmask(Len, 16), 16#FFFF, 16#FFFF, 16#FFFF}; + Len >= 48 -> {0, 0, 0, bmask(Len, 16), 16#FFFF, 16#FFFF, 16#FFFF, + 16#FFFF}; + Len >= 32 -> {0, 0, bmask(Len, 16), 16#FFFF, 16#FFFF, 16#FFFF, 16#FFFF, + 16#FFFF}; + Len >= 16 -> {0, bmask(Len, 16), 16#FFFF, 16#FFFF, 16#FFFF, 16#FFFF, + 16#FFFF, 16#FFFF}; + Len >= 0 -> {bmask(Len, 16), 16#FFFF, 16#FFFF, 16#FFFF, 16#FFFF, + 16#FFFF, 16#FFFF, 16#FFFF} + end. + +bmask(I, 8) when I >= 0, I =< 32 -> + 16#FF bsr (I rem 8); +bmask(I, 16) when I >= 0, I =< 128 -> + 16#FFFF bsr (I rem 16). + +calc_end_address(Addr, Len) -> + bor_with_mask(Addr, end_mask(Addr, Len)). + +bor_with_mask({A, B, C, D}, {E, F, G, H}) -> + {A bor E, B bor F, C bor G, D bor H}; +bor_with_mask({A, B, C, D, E, F, G, H}, {I, J, K, L, M, N, O, P}) -> + {A bor I, B bor J, C bor K, D bor L, E bor M, F bor N, G bor O, H bor P}. + +band_with_mask({A, B, C, D}, {E, F, G, H}) -> + {A band E, B band F, C band G, D band H}; +band_with_mask({A, B, C, D, E, F, G, H}, {I, J, K, L, M, N, O, P}) -> + {A band I, B band J, C band K, D band L, E band M, F band N, G band O, + H band P}. + +ip_lte({A, B, C, D1}, {A, B, C, D2}) -> D1 =< D2; +ip_lte({A, B, C1, _}, {A, B, C2, _}) -> C1 =< C2; +ip_lte({A, B1, _, _}, {A, B2, _, _}) -> B1 =< B2; +ip_lte({A1, _, _, _}, {A2, _, _, _}) -> A1 =< A2; +ip_lte({A, B, C, D, E, F, G, H1}, {A, B, C, D, E, F, G, H2}) -> H1 =< H2; +ip_lte({A, B, C, D, E, F, G1, _}, {A, B, C, D, E, F, G2, _}) -> G1 =< G2; +ip_lte({A, B, C, D, E, F1, _, _}, {A, B, C, D, E, F2, _, _}) -> F1 =< F2; +ip_lte({A, B, C, D, E1, _, _, _}, {A, B, C, D, E2, _, _, _}) -> E1 =< E2; +ip_lte({A, B, C, D1, _, _, _, _}, {A, B, C, D2, _, _, _, _}) -> D1 =< D2; +ip_lte({A, B, C1, _, _, _, _, _}, {A, B, C2, _, _, _, _, _}) -> C1 =< C2; +ip_lte({A, B1, _, _, _, _, _, _}, {A, B2, _, _, _, _, _, _}) -> B1 =< B2; +ip_lte({A1, _, _, _, _, _, _, _}, {A2, _, _, _, _, _, _, _}) -> A1 =< A2. + +ip_gte({A, B, C, D1}, {A, B, C, D2}) -> D1 >= D2; +ip_gte({A, B, C1, _}, {A, B, C2, _}) -> C1 >= C2; +ip_gte({A, B1, _, _}, {A, B2, _, _}) -> B1 >= B2; +ip_gte({A1, _, _, _}, {A2, _, _, _}) -> A1 >= A2; +ip_gte({A, B, C, D, E, F, G, H1}, {A, B, C, D, E, F, G, H2}) -> H1 >= H2; +ip_gte({A, B, C, D, E, F, G1, _}, {A, B, C, D, E, F, G2, _}) -> G1 >= G2; +ip_gte({A, B, C, D, E, F1, _, _}, {A, B, C, D, E, F2, _, _}) -> F1 >= F2; +ip_gte({A, B, C, D, E1, _, _, _}, {A, B, C, D, E2, _, _, _}) -> E1 >= E2; +ip_gte({A, B, C, D1, _, _, _, _}, {A, B, C, D2, _, _, _, _}) -> D1 >= D2; +ip_gte({A, B, C1, _, _, _, _, _}, {A, B, C2, _, _, _, _, _}) -> C1 >= C2; +ip_gte({A, B1, _, _, _, _, _, _}, {A, B2, _, _, _, _, _, _}) -> B1 >= B2; +ip_gte({A1, _, _, _, _, _, _, _}, {A2, _, _, _, _, _, _, _}) -> A1 >= A2. + +% @private Compare 2 CIDR specifications based on the following criteria: +% * IPv4 < IPv6 +% * If start range matches, sort on mask length +% * Otherwise, sort on start IP +cidr_lte({StartAddr, _, L1}, + {StartAddr, _, L2}) -> + L1 =< L2; +cidr_lte({StartAddr1, _, _L1}, + {StartAddr2, _, _L2}) when tuple_size(StartAddr1) =/= tuple_size(StartAddr2) -> + tuple_size(StartAddr1) =< tuple_size(StartAddr2); +cidr_lte({StartAddr1, _, _L1}, + {StartAddr2, _, _L2}) when tuple_size(StartAddr1) == tuple_size(StartAddr2) -> + ip_lte(StartAddr1, StartAddr2). + +%% @private merge a list of uniquely sorted CIDR blocks to their minimal +%% representation. +merge_sorted_cidrs(SortedCIDRs) -> + merge_sorted_cidrs(SortedCIDRs, []). + +merge_sorted_cidrs([], Acc) -> + lists:reverse(Acc); +merge_sorted_cidrs([CIDR], Acc) -> + lists:reverse([CIDR | Acc]); +merge_sorted_cidrs([CIDR1, CIDR2 | SortedCIDRs], Acc) -> + case contains(CIDR1, CIDR2) of + true -> + merge_sorted_cidrs([CIDR1 | SortedCIDRs], Acc); + false -> + merge_sorted_cidrs([CIDR2 | SortedCIDRs], [CIDR1 | Acc]) + end. diff --git a/test/hackney_cidr_tests.erl b/test/hackney_cidr_tests.erl new file mode 100644 index 00000000..dd9aa079 --- /dev/null +++ b/test/hackney_cidr_tests.erl @@ -0,0 +1,377 @@ +-module(hackney_cidr_tests). + +-include_lib("eunit/include/eunit.hrl"). + +parse_ipv4_test_() -> + [?_assertEqual({{0, 0, 0, 0}, {255, 255, 255, 255}, 0}, + hackney_cidr:parse("192.168.0.0/0", true)), + ?_assertEqual({{192, 0, 0, 0}, {192, 255, 255, 255}, 8}, + hackney_cidr:parse("192.168.0.0/8", true)), + ?_assertEqual({{192, 168, 0, 0}, {192, 169, 255, 255}, 15}, + hackney_cidr:parse("192.168.0.0/15", true)), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 255, 255}, 16}, + hackney_cidr:parse("192.168.0.0/16")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 127, 255}, 17}, + hackney_cidr:parse("192.168.0.0/17")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 63, 255}, 18}, + hackney_cidr:parse("192.168.0.0/18")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 31, 255}, 19}, + hackney_cidr:parse("192.168.0.0/19")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 15, 255}, 20}, + hackney_cidr:parse("192.168.0.0/20")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 7, 255}, 21}, + hackney_cidr:parse("192.168.0.0/21")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 3, 255}, 22}, + hackney_cidr:parse("192.168.0.0/22")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 1, 255}, 23}, + hackney_cidr:parse("192.168.0.0/23")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 0, 255}, 24}, + hackney_cidr:parse("192.168.0.0/24")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 0, 1}, 31}, + hackney_cidr:parse("192.168.0.0/31")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 0, 0}, 32}, + hackney_cidr:parse("192.168.0.0/32")), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 0, 0}, 32}, + hackney_cidr:parse(<<"192.168.0.0/32">>)), + ?_assertEqual({{192, 168, 0, 0}, {192, 168, 0, 0}, 32}, + hackney_cidr:parse(<<"192.168.0.0">>))]. + +parse_ipv6_test_() -> + [?_assertEqual({{0, 0, 0, 0, 0, 0, 0, 0}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}, + 0}, + hackney_cidr:parse("2001:abcd::/0", true)), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 65535, 65535, 65535, 65535, 65535, 65535}, + 32}, + hackney_cidr:parse("2001:abcd::/32")), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 32767, 65535, 65535, 65535, 65535, 65535}, + 33}, + hackney_cidr:parse("2001:abcd::/33")), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 16383, 65535, 65535, 65535, 65535, 65535}, + 34}, + hackney_cidr:parse("2001:abcd::/34")), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 8191, 65535, 65535, 65535, 65535, 65535}, + 35}, + hackney_cidr:parse("2001:abcd::/35")), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 4095, 65535, 65535, 65535, 65535, 65535}, + 36}, + hackney_cidr:parse("2001:abcd::/36")), + ?_assertEqual({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 0, 0, 0, 0, 0, 0}, + 128}, + hackney_cidr:parse("2001:abcd::/128")), + ?_assertEqual({{8193, 3512, 0, 0, 0, 0, 0, 0}, + {8193, 3512, 0, 65535, 65535, 65535, 65535, 65535}, + 48}, + hackney_cidr:parse("2001:db8::/48")), + ?_assertEqual({{8193, 3512, 0, 0, 0, 0, 0, 0}, + {8193, 3512, 0, 65535, 65535, 65535, 65535, 65535}, + 48}, + hackney_cidr:parse(<<"2001:db8::/48">>)), + ?_assertEqual({{8193,3512,0,0,0,0,0,1},{8193,3512,0,0,0,0,0,1},128}, + hackney_cidr:parse(<<"2001:0db8::1/128">>)), + ?_assertEqual({{8193,3512,0,0,0,0,0,1},{8193,3512,0,0,0,0,0,1},128}, + hackney_cidr:parse(<<"2001:0db8::1">>))] + . + + +to_string_test_() -> + [?_assertEqual("192.168.0.0/16", + hackney_cidr:to_string({{192, 168, 0, 0}, {192, 168, 255, 255}, 16})), + ?_assertEqual("2001:abcd::/32", + hackney_cidr:to_string({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 65535, 65535, 65535, 65535, 65535, 65535}, + 32}))]. + +to_binary_test_() -> + [?_assertEqual(<<"192.168.0.0/16">>, + hackney_cidr:to_binary({{192, 168, 0, 0}, {192, 168, 255, 255}, 16})), + ?_assertEqual(<<"2001:abcd::/32">>, + hackney_cidr:to_binary({{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 65535, 65535, 65535, 65535, 65535, 65535}, + 32}))]. + +ipv4_address_count_test_() -> + {ok, Addr} = inet:parse_address("192.168.0.0"), + [?_assertEqual(4294967296, hackney_cidr:address_count(Addr, 0)), + ?_assertEqual( 65536, hackney_cidr:address_count(Addr, 16)), + ?_assertEqual( 32768, hackney_cidr:address_count(Addr, 17)), + ?_assertEqual( 256, hackney_cidr:address_count(Addr, 24)), + ?_assertEqual( 1, hackney_cidr:address_count(Addr, 32))]. + +ipv6_address_count_test_() -> + {ok, Addr} = inet:parse_address("2001::abcd"), + [?_assertEqual(1 bsl 128, hackney_cidr:address_count(Addr, 0)), + ?_assertEqual(1 bsl 64, hackney_cidr:address_count(Addr, 64)), + ?_assertEqual(1, hackney_cidr:address_count(Addr, 128))]. + +contains_test_() -> + Block = hackney_cidr:parse("192.168.1.0/24"), + [?_assertNot(hackney_cidr:contains(Block, {}))]. + +ipv4_contains_test_() -> + Block = {{192, 168, 0, 0}, {192, 168, 255, 255}, 16}, + [?_assert(hackney_cidr:contains(Block, {192, 168, 0, 0})), + ?_assert(hackney_cidr:contains(Block, {192, 168, 0, 1})), + ?_assert(hackney_cidr:contains(Block, {192, 168, 1, 0})), + ?_assert(hackney_cidr:contains(Block, {192, 168, 0, 255})), + ?_assert(hackney_cidr:contains(Block, {192, 168, 255, 0})), + ?_assert(hackney_cidr:contains(Block, {192, 168, 255, 255})), + ?_assertNot(hackney_cidr:contains(Block, {192, 168, 255, 256})), + ?_assertNot(hackney_cidr:contains(Block, {192, 169, 0, 0})), + ?_assertNot(hackney_cidr:contains(Block, {192, 167, 255, 255}))]. + +ipv4_contains_cidr_test_() -> + Block = {{192, 168, 0, 0}, {192, 168, 255, 255}, 16}, + [?_assert(hackney_cidr:contains(Block, Block)), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("192.168.1.0/24"))), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("192.168.254.0/24"))), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("192.168.1.2/31"))), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("192.168.1.1/32"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("10.0.0.0/16"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("10.0.0.1/32"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("192.169.0.0/16"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("192.168.0.0/15")))]. + +ipv6_contains_test_() -> + Block = {{8193, 43981, 0, 0, 0, 0, 0, 0}, + {8193, 43981, 8191, 65535, 65535, 65535, 65535, 65535}, + 35}, + [?_assert(hackney_cidr:contains(Block, {8193, 43981, 0, 0, 0, 0, 0, 0})), + ?_assert(hackney_cidr:contains(Block, {8193, 43981, 0, 0, 0, 0, 0, 1})), + ?_assert(hackney_cidr:contains(Block, {8193, 43981, 8191, 65535, 65535, 65535, 65535, 65534})), + ?_assert(hackney_cidr:contains(Block, {8193, 43981, 8191, 65535, 65535, 65535, 65535, 65535})), + ?_assertNot(hackney_cidr:contains(Block, {8193, 43981, 8192, 65535, 65535, 65535, 65535, 65535})), + ?_assertNot(hackney_cidr:contains(Block, {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}))]. + +ipv6_contains_cidr_test_() -> + Block = hackney_cidr:parse("2001:abcd::/32"), + [?_assert(hackney_cidr:contains(Block, Block)), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("2001:abcd:1:1:1:1:1:1/128"))), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("2001:abcd:2000::/35"))), + ?_assert(hackney_cidr:contains(Block, hackney_cidr:parse("2001:abcd:2000:1:1:1::/96"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("2002:abcd::/35"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("2002:abcd:1:1:1:1:1:1/128"))), + ?_assertNot(hackney_cidr:contains(Block, hackney_cidr:parse("2001:ffff::/35")))]. + +usort_cidrs_test_() -> + [?_assertEqual([], hackney_cidr:usort_cidrs([])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("10.0.0.0/8")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("10.0.0.0/8")])), + ?_assertEqual([hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("2001:abcd::/32")])), + ?_assertEqual([hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("2001:abcd::/32")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("10.0.0.0/8")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("10.0.0.0/16"), + hackney_cidr:parse("10.1.0.0/16"), + hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("10.1.0.0/16"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("10.0.0.0/16"), + hackney_cidr:parse("10.0.0.0/8")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("2001:abcd:1::/48"), + hackney_cidr:parse("2001:abcd:1::/64"), + hackney_cidr:parse("2001:abcd:1::1/128")], + hackney_cidr:usort_cidrs([hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("2001:abcd:1::1/128"), + hackney_cidr:parse("2001:abcd:1::/64"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd:1::1/128"), + hackney_cidr:parse("2001:abcd:1::/48"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd:1::/64")]))]. + +merge_cidrs_test_() -> + [?_assertEqual([], hackney_cidr:merge_cidrs([])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/16"), + hackney_cidr:parse("10.10.0.0/16")], + hackney_cidr:merge_cidrs([hackney_cidr:parse("10.10.0.0/16"), + hackney_cidr:parse("10.0.0.1/32"), + hackney_cidr:parse("10.10.99.0/24"), + hackney_cidr:parse("10.0.99.0/24"), + hackney_cidr:parse("10.0.0.0/16")])), + ?_assertEqual([hackney_cidr:parse("10.0.1.1/32"), + hackney_cidr:parse("10.0.1.2/31"), + hackney_cidr:parse("10.0.1.4/30")], + hackney_cidr:merge_cidrs([hackney_cidr:parse("10.0.1.4/30"), + hackney_cidr:parse("10.0.1.4/32"), + hackney_cidr:parse("10.0.1.2/31"), + hackney_cidr:parse("10.0.1.1/32"), + hackney_cidr:parse("10.0.1.3/32"), + hackney_cidr:parse("10.0.1.4/30")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/16")], + hackney_cidr:merge_cidrs([hackney_cidr:parse("10.0.1.0/24"), + hackney_cidr:parse("10.0.254.0/24"), + hackney_cidr:parse("10.0.0.0/16"), + hackney_cidr:parse("10.0.100.99/32"), + hackney_cidr:parse("10.0.0.0/16")])), + ?_assertEqual([hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:merge_cidrs([hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("2001:abcd:1::1/128"), + hackney_cidr:parse("2001:abcd:1::/64"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd:1::1/128"), + hackney_cidr:parse("2001:abcd:1::/48"), + hackney_cidr:parse("10.0.0.0/8"), + hackney_cidr:parse("2001:abcd:1::/64")])), + ?_assertEqual([hackney_cidr:parse("2001:abcd::/32")], + hackney_cidr:merge_cidrs([hackney_cidr:parse("2001:abcd:abcd::/48"), + hackney_cidr:parse("2001:abcd:1234::/48"), + hackney_cidr:parse("2001:abcd:9999::/48"), + hackney_cidr:parse("2001:abcd::/32"), + hackney_cidr:parse("2001:abcd:abcd::1/128"), + hackney_cidr:parse("2001:abcd:abcd::4/126"), + hackney_cidr:parse("2001:abcd:abcd:0:0:abcd::/96")]))]. + +is_ipv4_test_() -> + {ok, Addr} = inet:parse_address("2001::abcd"), + [?_assert(hackney_cidr:is_ipv4({192, 168, 0, 0})), + ?_assertNot(hackney_cidr:is_ipv4({192, 168, 0, 256})), + ?_assertNot(hackney_cidr:is_ipv4({192, 168, 0})), + ?_assertNot(hackney_cidr:is_ipv4({192, 168, 0, 0, 0})), + ?_assertNot(hackney_cidr:is_ipv4(Addr))]. + +is_ipv6_test_() -> + [?_assert(hackney_cidr:is_ipv6({8193, 43981, 0, 0, 0, 0, 0, 0})), + ?_assertNot(hackney_cidr:is_ipv6({192, 168, 0, 0})), + ?_assertNot(hackney_cidr:is_ipv6({8193, 43981, 0, 0, 0, 0, 0, 70000})), + ?_assertNot(hackney_cidr:is_ipv6({8193, 43981, 0, 0, 0, 0, 0})), + ?_assertNot(hackney_cidr:is_ipv6({8193, 43981, 0, 0, 0, 0, 0, 0, 0}))]. + +ipv4_ip_lte_test_() -> + [?_assert(hackney_cidr:ip_lte({0, 0, 0, 0}, {0, 0, 0, 0})), + ?_assert(hackney_cidr:ip_lte({255, 255, 255, 255}, {255, 255, 255, 255})), + ?_assert(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 168, 1, 1})), + ?_assert(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 168, 1, 2})), + ?_assert(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 168, 2, 1})), + ?_assert(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 169, 1, 1})), + ?_assert(hackney_cidr:ip_lte({192, 168, 1, 1}, {193, 168, 1, 1})), + ?_assertNot(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 168, 1, 0})), + ?_assertNot(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 168, 0, 1})), + ?_assertNot(hackney_cidr:ip_lte({192, 168, 1, 1}, {192, 167, 1, 1})), + ?_assertNot(hackney_cidr:ip_lte({192, 168, 1, 1}, {191, 168, 1, 1}))]. + +ipv6_ip_lte_test_() -> + [?_assert(hackney_cidr:ip_lte({0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0})), + ?_assert(hackney_cidr:ip_lte({65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assert(hackney_cidr:ip_lte({0, 0, 0, 0, 0, 0, 0, 0}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 2, 2})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 3, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 4, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 5, 3, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 6, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 7, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43982, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8194, 43981, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 2}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 1, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 2, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 3, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 4, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 5, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43980, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_lte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8192, 43981, 6, 5, 4, 3, 2, 1}))]. + +ipv4_ip_gte_test_() -> + [?_assert(hackney_cidr:ip_gte({0, 0, 0, 0}, {0, 0, 0, 0})), + ?_assert(hackney_cidr:ip_gte({255, 255, 255, 255}, {255, 255, 255, 255})), + ?_assert(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 168, 1, 1})), + ?_assert(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 168, 1, 0})), + ?_assert(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 168, 0, 1})), + ?_assert(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 167, 1, 1})), + ?_assert(hackney_cidr:ip_gte({192, 168, 1, 1}, {191, 168, 1, 1})), + ?_assertNot(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 168, 1, 2})), + ?_assertNot(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 168, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({192, 168, 1, 1}, {192, 169, 1, 1})), + ?_assertNot(hackney_cidr:ip_gte({192, 168, 1, 1}, {193, 168, 1, 1}))]. + +ipv6_ip_gte_test_() -> + [?_assert(hackney_cidr:ip_gte({0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0})), + ?_assert(hackney_cidr:ip_gte({65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assert(hackney_cidr:ip_gte({65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 2}, + {8193, 43981, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 1, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 2, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 3, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 4, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 5, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43980, 6, 5, 4, 3, 2, 1})), + ?_assert(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8192, 43981, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({0, 0, 0, 0, 0, 0, 0, 0}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 2, 2})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 3, 3, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 4, 4, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 5, 5, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 6, 6, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43981, 7, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8193, 43982, 6, 5, 4, 3, 2, 1})), + ?_assertNot(hackney_cidr:ip_gte({8193, 43981, 6, 5, 4, 3, 2, 1}, + {8194, 43981, 6, 5, 4, 3, 2, 1}))]. From aa2b370fa57e0d422d70fe499e26a07df5b4731a Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 22 Feb 2024 02:22:25 +0100 Subject: [PATCH 02/13] handle no_proxy env variable This change allows hackney, to handle the no proxy environnement variable properly. This bbring real full support of proxy setup using environnement. --- include/hackney.hrl | 1 + rebar.lock | 9 ++-- src/hackney.erl | 128 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/include/hackney.hrl b/include/hackney.hrl index b26951ee..6c997f3b 100644 --- a/include/hackney.hrl +++ b/include/hackney.hrl @@ -59,3 +59,4 @@ -define(HTTP_PROXY_ENV_VARS, ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"]). -define(HTTPS_PROXY_ENV_VARS, ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"]). +-define(HTTP_NO_PROXY_ENV_VARS, ["no_proxy", "NO_PROXY"]). diff --git a/rebar.lock b/rebar.lock index 29b16ba8..0d63cac9 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,7 +1,6 @@ {"1.2.0", -[{<<"certifi">>,{pkg,<<"certifi">>,<<"2.12.0">>},0}, +[{<<"certifi">>,{pkg,<<"certifi">>,<<"2.13.0">>},0}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},0}, - {<<"inet_cidr">>,{pkg,<<"erl_cidr">>,<<"1.2.1">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},0}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},0}, @@ -9,18 +8,16 @@ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},0}]}. [ {pkg_hash,[ - {<<"certifi">>, <<"2D1CCA2EC95F59643862AF91F001478C9863C2AC9CB6E2F89780BFD8DE987329">>}, + {<<"certifi">>, <<"E52BE248590050B2DD33B0BB274B56678F9068E67805DCA8AA8B1CCDB016BBF6">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, - {<<"inet_cidr">>, <<"921BB463BCF10EAD937AE491E5BCAC2823E550B339A9893AC6574518553CC067">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, {pkg_hash_ext,[ - {<<"certifi">>, <<"EE68D85DF22E554040CDB4BE100F33873AC6051387BAF6A8F6CE82272340FF1C">>}, + {<<"certifi">>, <<"8F3D9533A0F06070AFDFD5D596B32E21C6580667A492891851B0E2737BC507A1">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, - {<<"inet_cidr">>, <<"FCF5D51B1CC1B26D1748AB39F9B614AB445AD17172A9396D48C6B5C15EA9EFCF">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, diff --git a/src/hackney.erl b/src/hackney.erl index 7dbef235..e7b2f134 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -676,14 +676,22 @@ maybe_proxy(Transport, Scheme, Host, Port, Options) end. maybe_proxy_from_env(Transport, _Scheme, Host, Port, Options, true) -> - ?report_debug("request without proxy", []), + ?report_debug("no proxy env is forced, request without proxy", []), hackney_connect:connect(Transport, Host, Port, Options, true); maybe_proxy_from_env(Transport, Scheme, Host, Port, Options, _) -> case get_proxy_env(Scheme) of {ok, Url} -> - proxy_from_url(Url, Transport, Host, Port, Options); + NoProxyEnv = get_no_proxy_env(), + case match_no_proxy_env(NoProxyEnv, Host) of + false -> + ?report_debug("request with proxy", [{proxy, Url}, {host, Host}]), + proxy_from_url(Url, Transport, Host, Port, Options); + true -> + ?report_debug("request without proxy", []), + hackney_connect:connect(Transport, Host, Port, Options, true) + end; false -> - ?report_debug("request without proxy", []), + ?report_debug("no proxy env setup, request without proxy", []), hackney_connect:connect(Transport, Host, Port, Options, true) end. @@ -705,17 +713,121 @@ proxy_from_url(Url, Transport, Host, Port, Options) -> end end. +get_no_proxy_env() -> + case application:get_env(hackney, no_proxy) of + undefined -> + case get_no_proxy_env(?HTTP_NO_PROXY_ENV_VARS) of + false -> + application:set_env(hackney, no_proxy, false), + false; + NoProxyEnv -> + parse_no_proxy_env(NoProxyEnv, []) + end; + {ok, NoProxyEnv} -> + NoProxyEnv + end. + +get_no_proxy_env([Key | Rest]) -> + case os:getenv(Key) of + false -> get_no_proxy_env(Rest); + NoProxyStr -> + lists:usort(string:split(NoProxyStr, ",")) + end; +get_no_proxy_env([]) -> + false. + +parse_no_proxy_env(["*" | _], _Acc) -> + application:set_env(hackney, no_proxy, '*'), + '*'; +parse_no_proxy_env([S | Rest], Acc) -> + try + CIDR = hackney_cidr:parse(S), + parse_no_proxy_env(Rest, [{cidr, CIDR} | Acc]) + catch + _:_ -> + Labels = string:tokens(S, "."), + parse_no_proxy_env(Rest, [{host, lists:reverse(Labels)}]) + end; +parse_no_proxy_env([], Acc) -> + NoProxy = lists:reverse(Acc), + application:set_env(hackney, no_proxy, NoProxy), + NoProxy. + +match_no_proxy_env(false, _Host) -> false; +match_no_proxy_env('*', _Host) -> true; +match_no_proxy_env(Patterns, Host) -> + do_match_no_proxy_env(Patterns, undefined, undefined, Host). + +do_match_no_proxy_env([{cidr, _CIDR} | _]=Patterns, undefined, Labels, Host) -> + Addrs = case inet:parse_address(Host) of + {ok, Addr} -> [Addr]; + _ -> getaddrs(Host) + end, + do_match_no_proxy_env(Patterns, Addrs, Labels, Host); +do_match_no_proxy_env([{cidr, CIDR} | Rest], Addrs, Labels, Host) -> + case test_host_cidr(Addrs, CIDR) of + true -> true; + false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host) + end; +do_match_no_proxy_env([{host, _Labels} | _] = Patterns, Addrs, undefined, Host) -> + HostLabels = string:tokens(Host, "."), + do_match_no_proxy_env(Patterns, Addrs, lists:reverse(HostLabels), Host); +do_match_no_proxy_env([{host, Labels} | Rest], Addrs, HostLabels, Host) -> + case test_host_labels(Labels, HostLabels) of + true -> true; + false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host) + end; +do_match_no_proxy_env([], _, _, _) -> + false. + +test_host_labels(["*" | R1], [_ | R2]) -> test_host_labels(R1, R2); +test_host_labels([ A | R1], [A | R2]) -> test_host_labels(R1, R2); +test_host_labels([], _) -> true; +test_host_labels(_, _) -> false. + +test_host_cidr([Addr, Rest], CIDR) -> + case hackney_cidr:contains(CIDR, Addr) of + true -> true; + false -> test_host_cidr(Rest, CIDR) + end; +test_host_cidr([], _) -> + false. + +getaddrs(Host) -> + IP4Addrs = case inet:getaddrs(Host, inet) of + {ok, Addrs} -> Addrs; + {error, nxdomain} -> [] + end, + case inet:getaddrs(Host, inet6) of + {ok, IP6Addrs} -> [IP6Addrs | IP4Addrs]; + {error, nxdomain} -> IP4Addrs + end. + get_proxy_env(https) -> - get_proxy_env(?HTTPS_PROXY_ENV_VARS); + case application:get_env(hackney, https_proxy) of + undefined -> + ProxyEnv = do_get_proxy_env(?HTTPS_PROXY_ENV_VARS), + application:set_env(hackney, https_proxy, ProxyEnv), + ProxyEnv; + {ok, Cached} -> + Cached + end; get_proxy_env(S) when S =:= http; S =:= http_unix -> - get_proxy_env(?HTTP_PROXY_ENV_VARS); + case application:get_env(hackney, http_proxy) of + undefined -> + ProxyEnv = do_get_proxy_env(?HTTP_PROXY_ENV_VARS), + application:set_env(hackney, http_proxy, ProxyEnv), + ProxyEnv; + {ok, Cached} -> + Cached + end. -get_proxy_env([Var | Rest]) -> +do_get_proxy_env([Var | Rest]) -> case os:getenv(Var) of - false -> get_proxy_env(Rest); + false -> do_get_proxy_env(Rest); Url -> {ok, Url} end; -get_proxy_env([]) -> +do_get_proxy_env([]) -> false. do_connect(ProxyHost, ProxyPort, undefined, Transport, Host, Port, Options) -> From 560d26664d7d9c01c6e191e243557cb2fe16f69a Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 7 May 2024 02:57:44 +0200 Subject: [PATCH 03/13] fix no_proxy env parsing --- src/hackney.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hackney.erl b/src/hackney.erl index e7b2f134..1323b2a6 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -731,7 +731,7 @@ get_no_proxy_env([Key | Rest]) -> case os:getenv(Key) of false -> get_no_proxy_env(Rest); NoProxyStr -> - lists:usort(string:split(NoProxyStr, ",")) + lists:usort(string:tokens(NoProxyStr, ",")) end; get_no_proxy_env([]) -> false. From bed3ae3cada41e542aa04e65655b60afa948f63d Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 28 May 2024 16:16:51 +0200 Subject: [PATCH 04/13] fix url parsing, handle IPs --- src/hackney.erl | 19 ++++++++++--------- src/hackney_happy.erl | 5 +++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/hackney.erl b/src/hackney.erl index 1323b2a6..4fdb8ec8 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -311,17 +311,18 @@ request(Method, #hackney_url{}=URL0, Headers0, Body, Options0) -> URL = hackney_url:normalize(URL0, PathEncodeFun), ?report_trace("request", [{method, Method}, - {url, URL}, - {headers, Headers0}, - {body, Body}, - {options, Options0}]), + {url, URL}, + {headers, Headers0}, + {body, Body}, + {options, Options0}]), #hackney_url{transport=Transport, - host = Host, - port = Port, - user = User, - password = Password, - scheme = Scheme} = URL, + host = Host, + port = Port, + user = User, + password = Password, + scheme = Scheme} = URL, + Options = case User of <<>> -> diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 0a90dc01..24e51f08 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -48,6 +48,11 @@ getaddrs(Hostname) -> getbyname(Hostname, Type) -> case (catch inet_res:getbyname(Hostname, Type)) of {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); + {error, nxdomain} = Error -> + case inet:parse_address(Hostname) of + {ok, IP} -> [IP]; + _ -> Error + end; {error, _Reason} -> []; Else -> %% ERLANG 22 has an issue when g matching somee DNS server messages From 420d855cfc507648576e00529a3d6c63f7de8edb Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 23 Aug 2024 23:45:24 +0200 Subject: [PATCH 05/13] getbyname: fix return when can't parse the address --- src/hackney_happy.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 24e51f08..a5e26069 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -51,7 +51,7 @@ getbyname(Hostname, Type) -> {error, nxdomain} = Error -> case inet:parse_address(Hostname) of {ok, IP} -> [IP]; - _ -> Error + _ -> [] end; {error, _Reason} -> []; Else -> From 033b181d4ae5cdc1e029b8afa39f8f34e113be05 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 21:10:03 +0200 Subject: [PATCH 06/13] fix IP parsing --- src/hackney_happy.erl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index a5e26069..0b1d99ca 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -11,7 +11,8 @@ connect(Hostname, Port, Opts) -> connect(Hostname, Port, Opts, ?CONNECT_TIMEOUT). -connect(Hostname, Port, Opts, Timeout) -> +connect(Hostname0, Port, Opts, Timeout) -> + Hostname = parse_address(Hostname0), case hackney_cidr:is_ipv6(Hostname) of true -> ?report_debug("connect using IPv6", [{hostname, Hostname}, {port, Port}]), @@ -39,6 +40,19 @@ connect(Hostname, Port, Opts, Timeout) -> end end end. + +parse_address(IPTuple) when is_tuple(IPTuple) -> IPTuple; +parse_address(IPBin) when is_binary(IPBin) -> + parse_address(binary_to_list(IPBin)); +%% IPv6 string with brackets +parse_address("[" ++ IPString) -> + parse_address(lists:sublist(IPString, length(IPString) - 1)); +parse_address(IPString) -> + case inet:parse_address(IPString) of + {ok, IP} -> IP; + {error, _} -> IPString + end. + getaddrs(Hostname) -> IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Hostname, 'aaaa')], @@ -48,7 +62,7 @@ getaddrs(Hostname) -> getbyname(Hostname, Type) -> case (catch inet_res:getbyname(Hostname, Type)) of {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); - {error, nxdomain} = Error -> + {error, nxdomain} -> case inet:parse_address(Hostname) of {ok, IP} -> [IP]; _ -> [] From c719c78f3daabd1082cb37a97390875754b2b75e Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 21:27:23 +0200 Subject: [PATCH 07/13] handle localhost --- src/hackney_happy.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 0b1d99ca..949068bd 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -59,6 +59,8 @@ getaddrs(Hostname) -> IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Hostname, 'a')], IP6Addrs ++ IP4Addrs. +getbyname("localhost", 'aaaa') -> [{0,0,0,0,0,0,0,1}]; +getbyname("localhost", 'a') -> [{127,0,0,1}]; getbyname(Hostname, Type) -> case (catch inet_res:getbyname(Hostname, Type)) of {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); From 1ec3761c2bd0ce67126769b754724feb239e4e92 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 21:37:15 +0200 Subject: [PATCH 08/13] fix export --- src/libs/hackney_cidr.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libs/hackney_cidr.erl b/src/libs/hackney_cidr.erl index b308f79d..e46d6183 100644 --- a/src/libs/hackney_cidr.erl +++ b/src/libs/hackney_cidr.erl @@ -15,6 +15,7 @@ -export([to_binary/1]). -export([is_ipv4/1]). -export([is_ipv6/1]). +-export([ip_gte/2, ip_lte/2]). -type cidr() :: {Start :: inet:ip4_address(), End :: inet:ip4_address(), MaskLen :: 0..32} | {Start :: inet:ip6_address(), End :: inet:ip6_address(), MaskLen :: 0..128}. From 231f0c2c0c701d1c695ccf860b72619c15ad1b5b Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 21:44:21 +0200 Subject: [PATCH 09/13] remove undefined func --- test/hackney_integration_tests_async_long_headers.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hackney_integration_tests_async_long_headers.erl b/test/hackney_integration_tests_async_long_headers.erl index c1ef5935..ae92f67a 100644 --- a/test/hackney_integration_tests_async_long_headers.erl +++ b/test/hackney_integration_tests_async_long_headers.erl @@ -34,7 +34,7 @@ start(#{status_code := StatusCode, method := Method}) -> stop(#{dummy_http_pid := Pid}, _Props) -> exit(Pid, normal), application:stop(hackney), - error_logger:tty(true), +% error_logger:tty(true), ok. From cf298cd7c99eb4fe8ef8ccb7baa7bd7d357af4d5 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 21:54:46 +0200 Subject: [PATCH 10/13] don't stop app at the endof the test --- test/hackney_integration_tests_async_long_headers.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hackney_integration_tests_async_long_headers.erl b/test/hackney_integration_tests_async_long_headers.erl index ae92f67a..a8077d00 100644 --- a/test/hackney_integration_tests_async_long_headers.erl +++ b/test/hackney_integration_tests_async_long_headers.erl @@ -33,8 +33,8 @@ start(#{status_code := StatusCode, method := Method}) -> stop(#{dummy_http_pid := Pid}, _Props) -> exit(Pid, normal), - application:stop(hackney), -% error_logger:tty(true), +% application:stop(hackney), + error_logger:tty(true), ok. From 8a45b03c6a90a874324395b18d91117a5623b1f1 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 22:54:55 +0200 Subject: [PATCH 11/13] match localhost directly --- src/hackney_happy.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 949068bd..1974bfec 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -53,14 +53,14 @@ parse_address(IPString) -> {error, _} -> IPString end. - + +getaddrs("localhost") -> + [{{0,0,0,0,0,0,0,1}, 'inet6'}, {{127,0,0,1}, 'inet'}]; getaddrs(Hostname) -> IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Hostname, 'aaaa')], IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Hostname, 'a')], IP6Addrs ++ IP4Addrs. -getbyname("localhost", 'aaaa') -> [{0,0,0,0,0,0,0,1}]; -getbyname("localhost", 'a') -> [{127,0,0,1}]; getbyname(Hostname, Type) -> case (catch inet_res:getbyname(Hostname, Type)) of {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); From 5c0f6e5a33acc0776a0e4d723259a20110e88643 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 23:03:48 +0200 Subject: [PATCH 12/13] address already parsed --- src/hackney_happy.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 1974bfec..1dd791aa 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -64,11 +64,6 @@ getaddrs(Hostname) -> getbyname(Hostname, Type) -> case (catch inet_res:getbyname(Hostname, Type)) of {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList); - {error, nxdomain} -> - case inet:parse_address(Hostname) of - {ok, IP} -> [IP]; - _ -> [] - end; {error, _Reason} -> []; Else -> %% ERLANG 22 has an issue when g matching somee DNS server messages From 782bc86365de5d506ab86c4ef07494d4e37fca76 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 10 Oct 2024 23:36:08 +0200 Subject: [PATCH 13/13] fix dialyzer warnings --- src/hackney_happy.erl | 47 ++++++++++++++++++++++++------------------- src/hackney_ssl.erl | 2 +- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/hackney_happy.erl b/src/hackney_happy.erl index 1dd791aa..5609da35 100644 --- a/src/hackney_happy.erl +++ b/src/hackney_happy.erl @@ -11,8 +11,10 @@ connect(Hostname, Port, Opts) -> connect(Hostname, Port, Opts, ?CONNECT_TIMEOUT). -connect(Hostname0, Port, Opts, Timeout) -> - Hostname = parse_address(Hostname0), +connect(Hostname, Port, Opts, Timeout) -> + do_connect(parse_address(Hostname), Port, Opts, Timeout). + +do_connect(Hostname, Port, Opts, Timeout) when is_tuple(Hostname) -> case hackney_cidr:is_ipv6(Hostname) of true -> ?report_debug("connect using IPv6", [{hostname, Hostname}, {port, Port}]), @@ -23,24 +25,27 @@ connect(Hostname0, Port, Opts, Timeout) -> ?report_debug("connect using IPv4", [{hostname, Hostname}, {port, Port}]), gen_tcp:connect(Hostname, Port, [inet | Opts], Timeout); false -> - ?report_debug("happy eyeballs, try to connect using IPv6", [{hostname, Hostname}, {port, Port}]), - Self = self(), - Addrs = getaddrs(Hostname), - Pid = spawn_link( fun() -> try_connect(Addrs, Port, Opts, Self, {error, nxdomain}) end), - MRef = erlang:monitor(process, Pid), - receive - {happy_connect, OK} -> - erlang:demonitor(MRef, [flush]), - OK; - {'DOWN', MRef, _Type, _Pid, Info} -> - {'error', {'connect_error', Info}} - after Timeout -> - erlang:demonitor(MRef, [flush]), - {error, connect_timeout} - end + {error, nxdomain} end + end; +do_connect(Hostname, Port, Opts, Timeout) -> + ?report_debug("happy eyeballs, try to connect using IPv6", [{hostname, Hostname}, {port, Port}]), + Self = self(), + Addrs = getaddrs(Hostname), + Pid = spawn_link( fun() -> try_connect(Addrs, Port, Opts, Self, {error, nxdomain}) end), + MRef = erlang:monitor(process, Pid), + receive + {happy_connect, OK} -> + erlang:demonitor(MRef, [flush]), + OK; + {'DOWN', MRef, _Type, _Pid, Info} -> + {'error', {'connect_error', Info}} + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error, connect_timeout} end. +-spec parse_address(inet:ip_address() | binary() | string()) -> inet:ip_address() | string(). parse_address(IPTuple) when is_tuple(IPTuple) -> IPTuple; parse_address(IPBin) when is_binary(IPBin) -> parse_address(binary_to_list(IPBin)); @@ -53,12 +58,12 @@ parse_address(IPString) -> {error, _} -> IPString end. - +-spec getaddrs(string()) -> [{inet:ip_address(), 'inet6' | 'inet'}]. getaddrs("localhost") -> [{{0,0,0,0,0,0,0,1}, 'inet6'}, {{127,0,0,1}, 'inet'}]; -getaddrs(Hostname) -> - IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Hostname, 'aaaa')], - IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Hostname, 'a')], +getaddrs(Name) -> + IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Name, 'aaaa')], + IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Name, 'a')], IP6Addrs ++ IP4Addrs. getbyname(Hostname, Type) -> diff --git a/src/hackney_ssl.erl b/src/hackney_ssl.erl index 79e1015b..e26a9b6e 100644 --- a/src/hackney_ssl.erl +++ b/src/hackney_ssl.erl @@ -123,7 +123,7 @@ find(_Fun, []) -> connect(Host, Port, Opts) -> - connect(Host, Port, Opts, infinity). + connect(Host, Port, Opts, 30000). connect(Host, Port, Opts0, Timeout) when is_list(Host), is_integer(Port), (Timeout =:= 5000 orelse is_integer(Timeout)) ->