From c7c81e102754b6bdd6833666f252e48f8f0611d1 Mon Sep 17 00:00:00 2001 From: Georgios Bitzes Date: Fri, 1 Apr 2016 17:03:14 +0200 Subject: [PATCH] DMC-816 #9 Implement .davixrc/.netrc parsing --- src/tools/CMakeLists.txt | 9 +- src/tools/davix_config_parser.cpp | 232 ++++++++++++++++++++++++ src/tools/davix_config_parser.hpp | 36 ++++ src/tools/davix_tool_params.cpp | 40 +++- test/unit/CMakeLists.txt | 3 +- test/unit/parser/test_config_parser.cpp | 126 +++++++++++++ 6 files changed, 435 insertions(+), 11 deletions(-) create mode 100644 src/tools/davix_config_parser.cpp create mode 100644 src/tools/davix_config_parser.hpp create mode 100644 test/unit/parser/test_config_parser.cpp diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index ca6b7d3e..87c2b16e 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -12,7 +12,14 @@ LIST(APPEND davix_rm_main_src "davix_tool_rm_main.cpp") LIST(APPEND davix_mkcol_main_src "davix_tool_mkcol_main.cpp") LIST(APPEND davix_mv_main_src "davix_tool_mv_main.cpp") LIST(APPEND davix_copy_main_src "davix_tool_copy_main.cpp") -LIST(APPEND davix_tool_common_src "${CMAKE_CURRENT_SOURCE_DIR}/davix_tool_params.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/davix_tool_util.cpp" "${SRC_SIMPLE_GET_PASS}" "${CMAKE_CURRENT_SOURCE_DIR}/davix_op.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/davix_taskqueue.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/davix_thread.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/davix_thread_pool.cpp") +LIST(APPEND davix_tool_common_src "${CMAKE_CURRENT_SOURCE_DIR}/davix_tool_params.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_tool_util.cpp" + "${SRC_SIMPLE_GET_PASS}" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_op.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_taskqueue.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_thread.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_thread_pool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/davix_config_parser.cpp") SET(davix_tool_common_src_up "${davix_tool_common_src}" PARENT_SCOPE) link_directories(${PROJECT_BINARY_DIR}/src/) diff --git a/src/tools/davix_config_parser.cpp b/src/tools/davix_config_parser.cpp new file mode 100644 index 00000000..dd377d0b --- /dev/null +++ b/src/tools/davix_config_parser.cpp @@ -0,0 +1,232 @@ +/* + * This File is part of Davix, The IO library for HTTP based protocols + * Copyright (C) CERN + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#include "davix_config_parser.hpp" +#include "davix_tool_util.hpp" +#include +#include +#include + +#define SSTR(message) static_cast(std::ostringstream().flush() << message).str() +static const std::string delimiters = " \t\n\"\'"; + +// find the end of a token - special handling for quotes +static size_t extract_end(const std::string &contents, const size_t start) { + if(contents[start] == '\"' || contents[start] == '\'') { + size_t end = contents.find(contents[start], start+1); + if(end == std::string::npos) return end; + end++; + + // hit into an escaped char? + if(contents[end-2] == '\\') { + return extract_end(contents, end-1); + } + return end; + } + else { + size_t end = contents.find_first_of(" \t\n", start); + if(end == std::string::npos) { + end = contents.size()+1; + } + return end; + } +} + +static void escape_token(std::string &s) { + // no need to escape + if(s[0] != '\'' && s[0] != '\"') { + return; + } + + std::string replacement(1, s[0]); + std::string target = "\\" + replacement; + + size_t index = 0; + while( (index = s.find(target, index)) != std::string::npos) { + s.replace(index, 2, replacement); + } + s = s.substr(1, s.size()-2); +} + +static size_t start_token(const std::string &contents, const size_t start=0) { + return contents.find_first_not_of(" \t\n", start); +} + +static bool next_token(const std::string &contents, std::string &err, size_t start, size_t &end, size_t &next) { + if(start == std::string::npos) { + return false; + } + // tokenize by splitting on delimiters + end = extract_end(contents, start); + if(end == std::string::npos) { + err = SSTR("Tokenization error (mismatched quote?) near position " << start << ":" << contents.substr(start)); + return false; + } + + next = contents.find_first_not_of(" \t\n", end); + return true; +} + +static size_t consume_macdef(const size_t start, const std::string &contents) { + return start_token(contents, contents.find("\n\n", start)); +} + +static void store_if_empty(std::string &source, const std::string &destination) { + if(source == "") + source = destination; +} + +static bool string_starts_with(const std::string &target, const std::string &prefix) { + if(target.size() < prefix.size()) return false; + return std::equal(prefix.begin(), prefix.end(), target.begin()); +} + +static void store_option(const std::string &first, const std::string &second, Davix::Tool::OptParams ¶ms) { + if(first == "login") { + store_if_empty(params.userlogpasswd.first, second); + } + else if(first == "password") { + store_if_empty(params.userlogpasswd.second, second); + } + else if(first == "cert") { + store_if_empty(params.cred_path, Davix::Tool::SanitiseTildedPath(second.c_str())); + } + else if(first == "key") { + store_if_empty(params.priv_key, Davix::Tool::SanitiseTildedPath(second.c_str())); + } + else if(first == "capath") { + params.params.addCertificateAuthorityPath(Davix::Tool::SanitiseTildedPath(second.c_str())); + } + else if(first == "s3accesskey") { + store_if_empty(params.aws_auth.second, second); + } + else if(first == "s3secretkey") { + store_if_empty(params.aws_auth.first, second); + } + else if(first == "s3region") { + store_if_empty(params.aws_region, second); + } + else if(first == "s3alternate") { + if(second == "true") params.aws_alternate = true; + } + else if(first == "s3token") { + store_if_empty(params.aws_token, second); + } + else if(first == "azurekey") { + store_if_empty(params.azure_key, second); + } +} + +namespace Davix { + +std::vector davix_config_tokenize(const std::string &contents, std::string &err) { + std::vector tokens; + size_t start = start_token(contents), end, next; + while(next_token(contents, err, start, end, next)) { + std::string token = contents.substr(start, end-start); + escape_token(token); + tokens.push_back(token); + + start = next; + } + return tokens; +} + +// returns true if there was any match for host +bool davix_config_apply(const std::string &filename, const std::string &contents, const Uri &uri, Tool::OptParams ¶ms) { + std::string err; + + std::string prevtoken; + std::string hostname; + std::string path; + + bool active = false; + bool in_default = false; + + size_t start = start_token(contents), end, next; + while(next_token(contents, err, start, end, next)) { + std::string token = contents.substr(start, end-start); + + // first token in the pair + if(prevtoken == "") { + // special case: ignore macro definitions + if(token == "macdef") { + start = consume_macdef(start, contents); + continue; + } + if(token == "default") { + if(active) return true; + + if(in_default) { + std::cerr << "davix: Warning: Malformed config file: " << filename << ". No entries should follow after 'default'." << std::endl; + return false; + } + active = true; + in_default = true; + path = ""; + std::cerr << "davix: using " << filename << " to load additional configuration. (match: default)" << std::endl; + } + else { + prevtoken = token; + } + } + // second token in the pair + else { + escape_token(token); + + if(prevtoken == "machine") { + if(active) return true; + path = ""; + + if(token == uri.getHost()) { + std::cerr << "davix: using " << filename << " to load additional configuration. (match: " << uri.getHost() << ")" << std::endl; + hostname = token; + active = true; + } + } + else if(prevtoken == "path") { + // only match a single explicit path! + if(active && path != "" && string_starts_with(uri.getPath(), path)) return true; + path = token; + } + else if(active && string_starts_with(uri.getPath(), path)) { + store_option(prevtoken, token, params); + } + prevtoken = ""; + } + start = next; + } + return active; +} + +bool davix_config_apply(const std::string &filename, Tool::OptParams ¶ms, const std::string &url) { + Uri uri(url); + std::string sanitized = Tool::SanitiseTildedPath(filename.c_str()); + std::ifstream t(sanitized); + if(uri.getHost() != "" && t.good()) { + std::string contents((std::istreambuf_iterator(t)), + std::istreambuf_iterator()); + + return davix_config_apply(filename, contents, uri, params); + } + return false; +} + +} diff --git a/src/tools/davix_config_parser.hpp b/src/tools/davix_config_parser.hpp new file mode 100644 index 00000000..e5a376a6 --- /dev/null +++ b/src/tools/davix_config_parser.hpp @@ -0,0 +1,36 @@ +/* + * This File is part of Davix, The IO library for HTTP based protocols + * Copyright (C) CERN + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef DAVIX_CONFIG_PARSER_HPP +#define DAVIX_CONFIG_PARSER_HPP + +#include +#include +#include "davix_tool_params.hpp" + +namespace Davix { + +std::vector davix_config_tokenize(const std::string &contents, std::string &err); +bool davix_config_apply(const std::string &filename, Tool::OptParams ¶ms, const std::string &url); +bool davix_config_apply(const std::string &filename, const std::string &contents, const Uri &uri, Tool::OptParams ¶ms); + +} + +#endif \ No newline at end of file diff --git a/src/tools/davix_tool_params.cpp b/src/tools/davix_tool_params.cpp index 3b88a5ee..616cf865 100644 --- a/src/tools/davix_tool_params.cpp +++ b/src/tools/davix_tool_params.cpp @@ -22,6 +22,7 @@ #include #include "davix_tool_params.hpp" #include "davix_tool_util.hpp" +#include "davix_config_parser.hpp" #include #include #include @@ -201,6 +202,11 @@ static struct timespec parse_timeout(const std::string & opt, char** argv){ return timelapse; } +void parse_davix_config(OptParams &p, std::string url) { + if(davix_config_apply("~/.davixrc", p, url)) return; + if(davix_config_apply("~/.netrc", p, url)) return; +} + int parse_davix_options_generic(const std::string &opt_filter, const struct option* long_options, int argc, char** argv, OptParams & p, DavixError** err){ @@ -386,9 +392,14 @@ int parse_davix_options(int argc, char** argv, OptParams & p, DavixError** err){ {0, 0, 0, 0 } }; - return parse_davix_options_generic(arg_tool_main, long_options, + if( parse_davix_options_generic(arg_tool_main, long_options, argc, argv, - p, err); + p, err) < 0) { + return -1; + } + + parse_davix_config(p, p.vec_arg[0]); + return 0; } @@ -408,6 +419,7 @@ int parse_davix_ls_options(int argc, char** argv, OptParams & p, DavixError** er option_abort(argv); return -1; } + parse_davix_config(p, p.vec_arg[0]); return 0; } @@ -431,6 +443,7 @@ int parse_davix_get_options(int argc, char** argv, OptParams & p, DavixError** e if(p.vec_arg.size() == 2){ p.output_file_path = p.vec_arg[1]; } + parse_davix_config(p, p.vec_arg[0]); return 0; } @@ -442,8 +455,6 @@ int parse_davix_put_options(int argc, char** argv, OptParams & p, DavixError** e {0, 0, 0, 0 } }; - - if( parse_davix_options_generic(arg_tool_main, long_options, argc, @@ -455,6 +466,7 @@ int parse_davix_put_options(int argc, char** argv, OptParams & p, DavixError** e return -1; } p.input_file_path = p.vec_arg[0]; + parse_davix_config(p, p.vec_arg[1]); return 0; } @@ -468,9 +480,14 @@ int parse_davix_copy_options(int argc, char** argv, OptParams & p, DavixError** {0, 0, 0, 0 } }; - return parse_davix_options_generic(arg_tool_main, long_options, + if(parse_davix_options_generic(arg_tool_main, long_options, argc, argv, - p, err); + p, err) < 0) { + return -1; + } + + parse_davix_config(p, p.vec_arg[0]); + return 0; } int parse_davix_rm_options(int argc, char** argv, OptParams & p, DavixError** err){ @@ -482,15 +499,20 @@ int parse_davix_rm_options(int argc, char** argv, OptParams & p, DavixError** er {0, 0, 0, 0 } }; - return parse_davix_options_generic(arg_tool_main, long_options, + if( parse_davix_options_generic(arg_tool_main, long_options, argc, argv, - p, err); + p, err) < 0) { + return -1; + } + + parse_davix_config(p, p.vec_arg[0]); + return 0; } std::string get_common_options(){ return " Common Options:\n" "\t--conn-timeout TIME: Connection timeout in seconds. default: 30\n" - "\t--retry NUMBER: Number of retry attempt in case of an operation failure. default: 10\n" + "\t--retry NUMBER: Number of retry attempts in case of an operation failure. default: 3\n" "\t--retry-delay TIME: Number of seconds to wait between retry attempts. default: 0\n" "\t--debug: Debug mode\n" "\t--header, -H: Add a header field to the request\n" diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 8878a334..ab7a4385 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -23,8 +23,9 @@ file(GLOB src_session session/*.cpp) file(GLOB src_parser xmlparser/*.cpp) file(GLOB src_files files/*.cpp) file(GLOB src_auth auth/*.cpp) +file(GLOB src_parser parser/*.cpp) + LIST(APPEND src_utils utils/utils_test.cpp ${SRC_STRING_UTILS_CPP}) -LIST(APPEND src_parser parser/parser_test.cpp) LIST(APPEND src_httprequest sessionfactory/test_factory.cpp) diff --git a/test/unit/parser/test_config_parser.cpp b/test/unit/parser/test_config_parser.cpp new file mode 100644 index 00000000..c640428e --- /dev/null +++ b/test/unit/parser/test_config_parser.cpp @@ -0,0 +1,126 @@ +#include +#include +#include + +using namespace Davix; + +TEST(ConfigParser, Tokenizer) { + std::vector tokens; + std::string err; + + tokens = davix_config_tokenize(" token1 \t\t\t\n\n token2 \t\t\n\n \" token3 in quotes and spaces \" ", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 3u); + ASSERT_EQ(tokens[0], "token1"); + ASSERT_EQ(tokens[1], "token2"); + ASSERT_EQ(tokens[2], " token3 in quotes and spaces "); + + tokens = davix_config_tokenize("\"single token in quotes\"", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 1u); + ASSERT_EQ(tokens[0], "single token in quotes"); + + // evil empty string + tokens = davix_config_tokenize("token1 \n\t\n \"\" token3 'token4 with single quotes @@!! ' ", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 4u); + ASSERT_EQ(tokens[0], "token1"); + ASSERT_EQ(tokens[1], ""); + ASSERT_EQ(tokens[2], "token3"); + ASSERT_EQ(tokens[3], "token4 with single quotes @@!! "); + + // escaped quotes inside arguments + tokens = davix_config_tokenize("\"\\\"\" \"token2 with \\\"escaped quotes\\\" \" ", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 2u); + ASSERT_EQ(tokens[0], "\""); + ASSERT_EQ(tokens[1], "token2 with \"escaped quotes\" "); + + // mixing quotes + tokens = davix_config_tokenize("'token1 with \" quotes' \"token2 with ' quotes\" 'token3 with \\' quotes' \"token4 with \\\" quotes\"", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 4u); + ASSERT_EQ(tokens[0], "token1 with \" quotes"); + ASSERT_EQ(tokens[1], "token2 with ' quotes"); + ASSERT_EQ(tokens[2], "token3 with ' quotes"); + ASSERT_EQ(tokens[3], "token4 with \" quotes"); + + // mismatched quote + tokens = davix_config_tokenize(" 'token1 \" ", err); + ASSERT_TRUE(err != ""); + ASSERT_EQ(tokens.size(), 0u); + err = ""; + + // mismatched quote + tokens = davix_config_tokenize(" \"token1 ' ", err); + ASSERT_TRUE(err != ""); + ASSERT_EQ(tokens.size(), 0u); + err = ""; + + // no extra whitespace + tokens = davix_config_tokenize("token1 token2", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 2u); + ASSERT_EQ(tokens[0], "token1"); + ASSERT_EQ(tokens[1], "token2"); + + // what a real file might look like + tokens = davix_config_tokenize("machine dpmhead-trunk.cern.ch\n certpath /tmp/x509up_u1000", err); + ASSERT_EQ(err, ""); + ASSERT_EQ(tokens.size(), 4u); + ASSERT_EQ(tokens[0], "machine"); + ASSERT_EQ(tokens[1], "dpmhead-trunk.cern.ch"); + ASSERT_EQ(tokens[2], "certpath"); + ASSERT_EQ(tokens[3], "/tmp/x509up_u1000"); +} + +TEST(ConfigParser, T1) { + std::string contents; + Uri uri("https://somehost/somepath/path2"); + Tool::OptParams params; + + // no match + contents = + "machine myhost\n" + " login \"mylogin\"\n" + " password \"mypass\"\n"; + ASSERT_FALSE(davix_config_apply("null", contents, uri, params)); + + // match + contents = + "machine somehost\n" + " login \"mylogin\"\n" + " password \"mypass\"\n"; + ASSERT_TRUE(davix_config_apply("null", contents, uri, params)); + ASSERT_EQ(params.userlogpasswd.first, "mylogin"); + ASSERT_EQ(params.userlogpasswd.second, "mypass"); + + // verify that existing settings are not overwritten + contents = + "machine somehost\n" + " login \"mylogin2\"\n" + " password \"mypass2\"\n"; + ASSERT_TRUE(davix_config_apply("null", contents, uri, params)); + ASSERT_EQ(params.userlogpasswd.first, "mylogin"); + ASSERT_EQ(params.userlogpasswd.second, "mypass"); + + // match for host, but not path + contents = + "machine somehost\n" + " path /someotherpath\n" + " s3accesskey key\n"; + ASSERT_TRUE(davix_config_apply("null", contents, uri, params)); + ASSERT_EQ(params.aws_auth.second, ""); + + // match for both host and path + generic host settings + contents = + "machine somehost\n" + " s3secretkey commonkey\n" + " path /someotherpath\n" + " s3accesskey key\n" + " path /somepath\n" + " s3accesskey correctkey\n"; + ASSERT_TRUE(davix_config_apply("null", contents, uri, params)); + ASSERT_EQ(params.aws_auth.second, "correctkey"); + ASSERT_EQ(params.aws_auth.first, "commonkey"); +}