From 72dcbe1c1b5204e2e92b80a25cac6acf54254ec2 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Tue, 29 Nov 2016 12:30:29 -0500 Subject: [PATCH 01/10] 1394: fixed handling of proxy username and password --- bootstrap | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/bootstrap b/bootstrap index c36fddba8..9190c6b1e 100755 --- a/bootstrap +++ b/bootstrap @@ -99,8 +99,9 @@ fetch({pkg, Name, Vsn}, App) -> {ok, Binary} -> {ok, Contents} = extract(Binary), ok = erl_tar:extract({binary, Contents}, [{cwd, Dir}, compressed]); - _ -> - io:format("Error: Unable to fetch package ~p ~p~n", [Name, Vsn]) + {error, {Reason, _}} -> + ReasonText = re:replace(atom_to_list(Reason), "_", " ", [global,{return,list}]), + io:format("Error: Unable to fetch package ~s ~s: ~s~n", [Name, Vsn, ReasonText]) end; true -> io:format("Dependency ~s already exists~n", [Name]) @@ -112,8 +113,10 @@ extract(Binary) -> {ok, Contents}. request(Url) -> + HttpOptions = [{relaxed, true} | get_proxy_auth()], + case httpc:request(get, {Url, []}, - [{relaxed, true}], + HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> @@ -402,3 +405,23 @@ otp_release1(Rel) -> binary:bin_to_list(Vsn, {0, Size - 1}) end end. + +%% extracts username and password from HTTPS_PROXY and returns them as tuple +get_proxy_auth() -> + get_proxy_auth(get_http_vars(https_proxy)). + +get_proxy_auth([]) -> + []; +get_proxy_auth(HttpsProxy) -> + {ok, {_, UserInfo, _, _, _, _}} = http_uri:parse(HttpsProxy), + parse_user_info(UserInfo). + +parse_user_info([]) -> + []; +parse_user_info(UserInfo) -> + Idx = string:chr(UserInfo, $:), + Username = string:sub_string(UserInfo, 1, Idx-1), + Password = string:sub_string(UserInfo, Idx+1), + %% password may contain url encoded characters, need to decode them first + [{proxy_auth, {Username, http_uri:decode(Password)}}]. + From ffc2cf98d390a35cf3f61c7c0bf5f26e3552fd2c Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Wed, 30 Nov 2016 13:45:06 -0500 Subject: [PATCH 02/10] 1394: added fix for rebar_utils, moved setting of http_options into init_config, added unit tests --- bootstrap | 29 +++++++++++++---------------- src/rebar3.erl | 3 ++- src/rebar_pkg_resource.erl | 4 +++- src/rebar_utils.erl | 24 +++++++++++++++++++++--- test/rebar_utils_SUITE.erl | 18 ++++++++++++++++-- 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/bootstrap b/bootstrap index 9190c6b1e..c7f0e0652 100755 --- a/bootstrap +++ b/bootstrap @@ -150,8 +150,9 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), - httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), + set_proxy_auth(UserInfo). compile(App, FirstFiles) -> Dir = filename:join(filename:absname("_build/default/lib/"), App), @@ -406,22 +407,18 @@ otp_release1(Rel) -> end end. -%% extracts username and password from HTTPS_PROXY and returns them as tuple -get_proxy_auth() -> - get_proxy_auth(get_http_vars(https_proxy)). - -get_proxy_auth([]) -> - []; -get_proxy_auth(HttpsProxy) -> - {ok, {_, UserInfo, _, _, _, _}} = http_uri:parse(HttpsProxy), - parse_user_info(UserInfo). - -parse_user_info([]) -> - []; -parse_user_info(UserInfo) -> +set_proxy_auth([]) -> + ok; +set_proxy_auth(UserInfo) -> Idx = string:chr(UserInfo, $:), Username = string:sub_string(UserInfo, 1, Idx-1), Password = string:sub_string(UserInfo, Idx+1), %% password may contain url encoded characters, need to decode them first - [{proxy_auth, {Username, http_uri:decode(Password)}}]. + put(proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + +get_proxy_auth() -> + case get(proxy_auth) of + undefined -> []; + ProxyAuth -> ProxyAuth + end. diff --git a/src/rebar3.erl b/src/rebar3.erl index 4e7e284db..1059904a2 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -145,6 +145,8 @@ run_aux(State, RawArgs) -> rebar_core:init_command(rebar_state:command_args(State10, Args), Task). init_config() -> + rebar_utils:set_httpc_options(), + %% Initialize logging system Verbosity = log_level(), ok = rebar_log:init(command_line, Verbosity), @@ -320,7 +322,6 @@ ensure_running(App, Caller) -> end. state_from_global_config(Config, GlobalConfigFile) -> - rebar_utils:set_httpc_options(), GlobalConfigTerms = rebar_config:consult_file(GlobalConfigFile), GlobalConfig = rebar_state:new(GlobalConfigTerms), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817cb..8e1713d11 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -107,8 +107,10 @@ make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. request(Url, ETag) -> + HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, - [{ssl, ssl_opts(Url)}, {relaxed, true}], + HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index f55f40fc8..4b4391148 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -70,7 +70,9 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1]). + reread_config/1, + get_proxy_auth/0, + set_proxy_auth/1]). %% for internal use only -export([otp_release/0]). @@ -825,8 +827,9 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), - httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), + set_proxy_auth(UserInfo). url_append_path(Url, ExtraPath) -> case http_uri:parse(Url) of @@ -865,3 +868,18 @@ list_dir(Dir) -> true -> file:list_dir_all(Dir); false -> file:list_dir(Dir) end. + +set_proxy_auth([]) -> + ok; +set_proxy_auth(UserInfo) -> + Idx = string:chr(UserInfo, $:), + Username = string:sub_string(UserInfo, 1, Idx-1), + Password = string:sub_string(UserInfo, Idx+1), + %% password may contain url encoded characters, need to decode them first + application:set_env(rebar, proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + +get_proxy_auth() -> + case application:get_env(rebar, proxy_auth) of + undefined -> []; + {ok, ProxyAuth} -> ProxyAuth + end. diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index b32992d35..0d496a31c 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -31,7 +31,8 @@ nonblacklisted_otp_version/1, blacklisted_otp_version/1, sh_does_not_miss_messages/1, - tup_merge/1]). + tup_merge/1, + proxy_auth/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -46,7 +47,8 @@ end_per_testcase(_, _Config) -> all() -> [{group, args_to_tasks}, sh_does_not_miss_messages, - tup_merge]. + tup_merge, + proxy_auth]. groups() -> [{args_to_tasks, [], [empty_arglist, @@ -272,3 +274,15 @@ tup_merge(_Config) -> rebar_utils:tup_sort([{a,a},{a,a,a},a,{b,a,a},b,{z,a},{z,a,a},{b,a},z]) ) ). + +proxy_auth(_Config) -> + %% proxy auth with regular username/password + rebar_utils:set_proxy_auth("Username", "Password"), + ?assertEqual([{proxy_auth, {"Username", "Password"}}], + rebar_utils:get_proxy_auth()), + %% proxy auth with username missing and url encoded password + rebar_utils:set_proxy_auth("", "?!abc#$"), + ?assertEqual([{proxy_auth, {"", "%3F!abc%23%24"}}], + rebar_utils:get_proxy_auth()). + + From 8a546cae8eab79ea2107476e0978247457d89acb Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Wed, 30 Nov 2016 13:48:22 -0500 Subject: [PATCH 03/10] 1394: fixed typo --- test/rebar_utils_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index 0d496a31c..6cbac1afb 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -277,11 +277,11 @@ tup_merge(_Config) -> proxy_auth(_Config) -> %% proxy auth with regular username/password - rebar_utils:set_proxy_auth("Username", "Password"), + rebar_utils:set_proxy_auth("Username:Password"), ?assertEqual([{proxy_auth, {"Username", "Password"}}], rebar_utils:get_proxy_auth()), %% proxy auth with username missing and url encoded password - rebar_utils:set_proxy_auth("", "?!abc#$"), + rebar_utils:set_proxy_auth(":?!abc#$"), ?assertEqual([{proxy_auth, {"", "%3F!abc%23%24"}}], rebar_utils:get_proxy_auth()). From f8873147abe49baa4e692b6dd93be1b8db89e172 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Wed, 30 Nov 2016 13:52:23 -0500 Subject: [PATCH 04/10] 1394: fixed typos --- test/rebar_utils_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index 6cbac1afb..ad37e002a 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -281,8 +281,8 @@ proxy_auth(_Config) -> ?assertEqual([{proxy_auth, {"Username", "Password"}}], rebar_utils:get_proxy_auth()), %% proxy auth with username missing and url encoded password - rebar_utils:set_proxy_auth(":?!abc#$"), - ?assertEqual([{proxy_auth, {"", "%3F!abc%23%24"}}], + rebar_utils:set_proxy_auth(":%3F!abc%23%24"), + ?assertEqual([{proxy_auth, {"", "?!abc#$"}}], rebar_utils:get_proxy_auth()). From e60562fb307a8ff59f56746427e60ed3b2a1227a Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 02:49:53 -0500 Subject: [PATCH 05/10] 1394: one more test --- test/rebar_utils_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index ad37e002a..e8f33a338 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -276,6 +276,9 @@ tup_merge(_Config) -> ). proxy_auth(_Config) -> + application:unset_env(rebar, proxy_auth), + %% proxy auth not set + ?assertEqual([], rebar_utils:get_proxy_auth()), %% proxy auth with regular username/password rebar_utils:set_proxy_auth("Username:Password"), ?assertEqual([{proxy_auth, {"Username", "Password"}}], From c0184eae706f6c2bf7bcf8fc1db03c0569cb73c1 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 10:27:07 -0500 Subject: [PATCH 06/10] 1394: refined export list and tests --- src/rebar_utils.erl | 4 ++-- test/rebar_utils_SUITE.erl | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 4b4391148..a0a44d19e 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -71,8 +71,8 @@ list_dir/1, user_agent/0, reread_config/1, - get_proxy_auth/0, - set_proxy_auth/1]). + get_proxy_auth/0]). + %% for internal use only -export([otp_release/0]). diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index e8f33a338..6307b4244 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -276,15 +276,20 @@ tup_merge(_Config) -> ). proxy_auth(_Config) -> + Host = "host:", + Port = "1234", + application:unset_env(rebar, proxy_auth), %% proxy auth not set ?assertEqual([], rebar_utils:get_proxy_auth()), %% proxy auth with regular username/password - rebar_utils:set_proxy_auth("Username:Password"), + os:putenv("http_proxy", "http://Username:Password@" ++ Host ++ Port), + rebar_utils:set_httpc_options(), ?assertEqual([{proxy_auth, {"Username", "Password"}}], rebar_utils:get_proxy_auth()), %% proxy auth with username missing and url encoded password - rebar_utils:set_proxy_auth(":%3F!abc%23%24"), + os:putenv("http_proxy", "http://:%3F!abc%23%24@" ++ Host ++ Port), + rebar_utils:set_httpc_options(), ?assertEqual([{proxy_auth, {"", "?!abc#$"}}], rebar_utils:get_proxy_auth()). From 603683bca1ec067e7490405fd308cdabbc442702 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 11:16:33 -0500 Subject: [PATCH 07/10] 1394: restore original proxy spec after tests --- test/rebar_utils_SUITE.erl | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index 6307b4244..bdbffb0d4 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -278,19 +278,34 @@ tup_merge(_Config) -> proxy_auth(_Config) -> Host = "host:", Port = "1234", + + proxy_auth(_Config, "http_proxy"), + proxy_auth(_Config, "https_proxy"). + +proxy_auth(_Config, ProxyEnvKey) -> + %% remember current proxy specification + OldProxySpec = os:getenv(ProxyEnvKey), - application:unset_env(rebar, proxy_auth), %% proxy auth not set + application:unset_env(rebar, proxy_auth), ?assertEqual([], rebar_utils:get_proxy_auth()), + %% proxy auth with regular username/password - os:putenv("http_proxy", "http://Username:Password@" ++ Host ++ Port), + os:putenv(ProxyEnvKey, "http://Username:Password@" ++ Host ++ Port), rebar_utils:set_httpc_options(), ?assertEqual([{proxy_auth, {"Username", "Password"}}], rebar_utils:get_proxy_auth()), + %% proxy auth with username missing and url encoded password - os:putenv("http_proxy", "http://:%3F!abc%23%24@" ++ Host ++ Port), + os:putenv(ProxyEnvKey, "http://:%3F!abc%23%24@" ++ Host ++ Port), rebar_utils:set_httpc_options(), ?assertEqual([{proxy_auth, {"", "?!abc#$"}}], - rebar_utils:get_proxy_auth()). - - + rebar_utils:get_proxy_auth()), + + %% restore original proxy specification if any + restore_proxy_env(OldProxySpec). + +restore_proxy_env(false) -> + ok; +restore_proxy_env(ProxySpec) -> + os:putenv("http_proxy", ProxySpec). From 9ace3ba9fc1b2a1a7096e4fe62d9867f737cff1d Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 11:17:07 -0500 Subject: [PATCH 08/10] 1394: fixed typo --- test/rebar_utils_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index bdbffb0d4..57b64749c 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -276,13 +276,13 @@ tup_merge(_Config) -> ). proxy_auth(_Config) -> - Host = "host:", - Port = "1234", - proxy_auth(_Config, "http_proxy"), proxy_auth(_Config, "https_proxy"). proxy_auth(_Config, ProxyEnvKey) -> + Host = "host:", + Port = "1234", + %% remember current proxy specification OldProxySpec = os:getenv(ProxyEnvKey), From 8df95d53bfd5dc892e69364dfe87ecf95c6897a2 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 11:20:58 -0500 Subject: [PATCH 09/10] 1394: fixed typo --- test/rebar_utils_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index 57b64749c..dd92bbf54 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -303,9 +303,9 @@ proxy_auth(_Config, ProxyEnvKey) -> rebar_utils:get_proxy_auth()), %% restore original proxy specification if any - restore_proxy_env(OldProxySpec). + restore_proxy_env(ProxyEnvKey, OldProxySpec). -restore_proxy_env(false) -> +restore_proxy_env(_, false) -> ok; -restore_proxy_env(ProxySpec) -> - os:putenv("http_proxy", ProxySpec). +restore_proxy_env(ProxyEnvKey, ProxySpec) -> + os:putenv(ProxyEnvKey, ProxySpec). From 2c155ead23abeedb39ff761e4db5269f7b2a78ca Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 1 Dec 2016 11:47:05 -0500 Subject: [PATCH 10/10] Fully clean up after test utils for proxy --- test/rebar_utils_SUITE.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index dd92bbf54..8b8769bf4 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -303,9 +303,10 @@ proxy_auth(_Config, ProxyEnvKey) -> rebar_utils:get_proxy_auth()), %% restore original proxy specification if any - restore_proxy_env(ProxyEnvKey, OldProxySpec). + restore_proxy_env(ProxyEnvKey, OldProxySpec), + application:unset_env(rebar, proxy_auth). -restore_proxy_env(_, false) -> - ok; +restore_proxy_env(ProxyEnvKey, false) -> + os:putenv(ProxyEnvKey, ""); restore_proxy_env(ProxyEnvKey, ProxySpec) -> - os:putenv(ProxyEnvKey, ProxySpec). + os:putenv(ProxyEnvKey, ProxySpec).