From 0b43ae85f99db07b0b5c71271998bb5d7f4dee3d Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 22 Feb 2024 02:22:25 +0100 Subject: [PATCH] handle no_proxy env variable This change allows hackney, to handle the no proxy environnement variable properly. This bbring real full support of proxy setup using environnement. --- include/hackney.hrl | 1 + rebar.lock | 9 ++-- src/hackney.erl | 128 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/include/hackney.hrl b/include/hackney.hrl index b26951ee..6c997f3b 100644 --- a/include/hackney.hrl +++ b/include/hackney.hrl @@ -59,3 +59,4 @@ -define(HTTP_PROXY_ENV_VARS, ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"]). -define(HTTPS_PROXY_ENV_VARS, ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"]). +-define(HTTP_NO_PROXY_ENV_VARS, ["no_proxy", "NO_PROXY"]). diff --git a/rebar.lock b/rebar.lock index 29b16ba8..0d63cac9 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,7 +1,6 @@ {"1.2.0", -[{<<"certifi">>,{pkg,<<"certifi">>,<<"2.12.0">>},0}, +[{<<"certifi">>,{pkg,<<"certifi">>,<<"2.13.0">>},0}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},0}, - {<<"inet_cidr">>,{pkg,<<"erl_cidr">>,<<"1.2.1">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},0}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},0}, @@ -9,18 +8,16 @@ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},0}]}. [ {pkg_hash,[ - {<<"certifi">>, <<"2D1CCA2EC95F59643862AF91F001478C9863C2AC9CB6E2F89780BFD8DE987329">>}, + {<<"certifi">>, <<"E52BE248590050B2DD33B0BB274B56678F9068E67805DCA8AA8B1CCDB016BBF6">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, - {<<"inet_cidr">>, <<"921BB463BCF10EAD937AE491E5BCAC2823E550B339A9893AC6574518553CC067">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, {pkg_hash_ext,[ - {<<"certifi">>, <<"EE68D85DF22E554040CDB4BE100F33873AC6051387BAF6A8F6CE82272340FF1C">>}, + {<<"certifi">>, <<"8F3D9533A0F06070AFDFD5D596B32E21C6580667A492891851B0E2737BC507A1">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, - {<<"inet_cidr">>, <<"FCF5D51B1CC1B26D1748AB39F9B614AB445AD17172A9396D48C6B5C15EA9EFCF">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, diff --git a/src/hackney.erl b/src/hackney.erl index 7dbef235..e7b2f134 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -676,14 +676,22 @@ maybe_proxy(Transport, Scheme, Host, Port, Options) end. maybe_proxy_from_env(Transport, _Scheme, Host, Port, Options, true) -> - ?report_debug("request without proxy", []), + ?report_debug("no proxy env is forced, request without proxy", []), hackney_connect:connect(Transport, Host, Port, Options, true); maybe_proxy_from_env(Transport, Scheme, Host, Port, Options, _) -> case get_proxy_env(Scheme) of {ok, Url} -> - proxy_from_url(Url, Transport, Host, Port, Options); + NoProxyEnv = get_no_proxy_env(), + case match_no_proxy_env(NoProxyEnv, Host) of + false -> + ?report_debug("request with proxy", [{proxy, Url}, {host, Host}]), + proxy_from_url(Url, Transport, Host, Port, Options); + true -> + ?report_debug("request without proxy", []), + hackney_connect:connect(Transport, Host, Port, Options, true) + end; false -> - ?report_debug("request without proxy", []), + ?report_debug("no proxy env setup, request without proxy", []), hackney_connect:connect(Transport, Host, Port, Options, true) end. @@ -705,17 +713,121 @@ proxy_from_url(Url, Transport, Host, Port, Options) -> end end. +get_no_proxy_env() -> + case application:get_env(hackney, no_proxy) of + undefined -> + case get_no_proxy_env(?HTTP_NO_PROXY_ENV_VARS) of + false -> + application:set_env(hackney, no_proxy, false), + false; + NoProxyEnv -> + parse_no_proxy_env(NoProxyEnv, []) + end; + {ok, NoProxyEnv} -> + NoProxyEnv + end. + +get_no_proxy_env([Key | Rest]) -> + case os:getenv(Key) of + false -> get_no_proxy_env(Rest); + NoProxyStr -> + lists:usort(string:split(NoProxyStr, ",")) + end; +get_no_proxy_env([]) -> + false. + +parse_no_proxy_env(["*" | _], _Acc) -> + application:set_env(hackney, no_proxy, '*'), + '*'; +parse_no_proxy_env([S | Rest], Acc) -> + try + CIDR = hackney_cidr:parse(S), + parse_no_proxy_env(Rest, [{cidr, CIDR} | Acc]) + catch + _:_ -> + Labels = string:tokens(S, "."), + parse_no_proxy_env(Rest, [{host, lists:reverse(Labels)}]) + end; +parse_no_proxy_env([], Acc) -> + NoProxy = lists:reverse(Acc), + application:set_env(hackney, no_proxy, NoProxy), + NoProxy. + +match_no_proxy_env(false, _Host) -> false; +match_no_proxy_env('*', _Host) -> true; +match_no_proxy_env(Patterns, Host) -> + do_match_no_proxy_env(Patterns, undefined, undefined, Host). + +do_match_no_proxy_env([{cidr, _CIDR} | _]=Patterns, undefined, Labels, Host) -> + Addrs = case inet:parse_address(Host) of + {ok, Addr} -> [Addr]; + _ -> getaddrs(Host) + end, + do_match_no_proxy_env(Patterns, Addrs, Labels, Host); +do_match_no_proxy_env([{cidr, CIDR} | Rest], Addrs, Labels, Host) -> + case test_host_cidr(Addrs, CIDR) of + true -> true; + false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host) + end; +do_match_no_proxy_env([{host, _Labels} | _] = Patterns, Addrs, undefined, Host) -> + HostLabels = string:tokens(Host, "."), + do_match_no_proxy_env(Patterns, Addrs, lists:reverse(HostLabels), Host); +do_match_no_proxy_env([{host, Labels} | Rest], Addrs, HostLabels, Host) -> + case test_host_labels(Labels, HostLabels) of + true -> true; + false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host) + end; +do_match_no_proxy_env([], _, _, _) -> + false. + +test_host_labels(["*" | R1], [_ | R2]) -> test_host_labels(R1, R2); +test_host_labels([ A | R1], [A | R2]) -> test_host_labels(R1, R2); +test_host_labels([], _) -> true; +test_host_labels(_, _) -> false. + +test_host_cidr([Addr, Rest], CIDR) -> + case hackney_cidr:contains(CIDR, Addr) of + true -> true; + false -> test_host_cidr(Rest, CIDR) + end; +test_host_cidr([], _) -> + false. + +getaddrs(Host) -> + IP4Addrs = case inet:getaddrs(Host, inet) of + {ok, Addrs} -> Addrs; + {error, nxdomain} -> [] + end, + case inet:getaddrs(Host, inet6) of + {ok, IP6Addrs} -> [IP6Addrs | IP4Addrs]; + {error, nxdomain} -> IP4Addrs + end. + get_proxy_env(https) -> - get_proxy_env(?HTTPS_PROXY_ENV_VARS); + case application:get_env(hackney, https_proxy) of + undefined -> + ProxyEnv = do_get_proxy_env(?HTTPS_PROXY_ENV_VARS), + application:set_env(hackney, https_proxy, ProxyEnv), + ProxyEnv; + {ok, Cached} -> + Cached + end; get_proxy_env(S) when S =:= http; S =:= http_unix -> - get_proxy_env(?HTTP_PROXY_ENV_VARS); + case application:get_env(hackney, http_proxy) of + undefined -> + ProxyEnv = do_get_proxy_env(?HTTP_PROXY_ENV_VARS), + application:set_env(hackney, http_proxy, ProxyEnv), + ProxyEnv; + {ok, Cached} -> + Cached + end. -get_proxy_env([Var | Rest]) -> +do_get_proxy_env([Var | Rest]) -> case os:getenv(Var) of - false -> get_proxy_env(Rest); + false -> do_get_proxy_env(Rest); Url -> {ok, Url} end; -get_proxy_env([]) -> +do_get_proxy_env([]) -> false. do_connect(ProxyHost, ProxyPort, undefined, Transport, Host, Port, Options) ->