From b36bd959a32eeb2b94dde609a2087f2b7ff92f54 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Tue, 19 Mar 2024 19:57:53 +0100 Subject: [PATCH] Added s3_storage backend (WIP) --- .github/workflows/cmake.yml | 89 ++++++++++++++++++----- CMakeLists.txt | 7 ++ src/app.cpp | 3 + src/binsrv/basic_storage_backend.cpp | 4 + src/binsrv/basic_storage_backend.hpp | 4 + src/binsrv/filesystem_storage_backend.cpp | 5 ++ src/binsrv/filesystem_storage_backend.hpp | 2 + src/binsrv/s3_storage_backend.cpp | 70 ++++++++++++++++++ src/binsrv/s3_storage_backend.hpp | 42 +++++++++++ src/binsrv/storage_backend_factory.cpp | 12 ++- 10 files changed, 217 insertions(+), 21 deletions(-) create mode 100644 src/binsrv/s3_storage_backend.cpp create mode 100644 src/binsrv/s3_storage_backend.hpp diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a97157c..5533d91 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -7,6 +7,9 @@ on: branches: [ "main" ] env: + AWS_SDK_CPP_MAJOR: 1 + AWS_SDK_CPP_MINOR: 11 + AWS_SDK_CPP_PATCH: 286 BOOST_MAJOR: 1 BOOST_MINOR: 83 BOOST_PATCH: 0 @@ -17,7 +20,8 @@ jobs: runs-on: ubuntu-22.04 name: Formatting checks steps: - - uses: actions/checkout@v4 + - name: Cheking out code + uses: actions/checkout@v4 with: fetch-depth: 2 @@ -52,7 +56,7 @@ jobs: build_type: "Debug", cc: "gcc-13", cxx: "g++-13", - label: "debug_gcc13", + label: "Debug-gcc13", run_mtr: true } - { @@ -60,7 +64,7 @@ jobs: build_type: "RelWithDebInfo", cc: "gcc-13", cxx: "g++-13", - label: "relwithdebinfo_gcc13", + label: "RelWithDebInfo-gcc13", run_mtr: true } - { @@ -69,7 +73,8 @@ jobs: cc: "gcc-13", cxx: "g++-13", sanitizer_cmake_flags: "-DWITH_ASAN=ON", - label: "asan_gcc13", + aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON", + label: "ASan-gcc13", run_mtr: true, mtr_options: "--sanitize" } @@ -79,7 +84,8 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", - label: "debug_clang17", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", + label: "Debug-clang17", run_clang_tidy: true, } - { @@ -88,7 +94,8 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", - label: "relwithdebinfo_clang17", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", + label: "RelWithDebInfo-clang17", run_clang_tidy: true, } - { @@ -97,8 +104,10 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", sanitizer_cmake_flags: "-DWITH_ASAN=ON", - label: "asan_clang17", + aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON", + label: "ASan-clang17", run_mtr: true, mtr_options: "--sanitize" } @@ -117,10 +126,15 @@ jobs: - name: Info df run: df -h - - name: Install MySQL client libraries + - name: Info GitHub directories + run: | + echo github.workspace: ${{github.workspace}} + echo runner.temp : ${{runner.temp}} + + - name: Install MySQL client libraries and CURL Development libraries run: | sudo apt-get update - sudo apt-get install libmysqlclient-dev + sudo apt-get install libmysqlclient-dev libcurl4-openssl-dev - name: Install MySQL server and MTR if: matrix.config.run_mtr @@ -156,7 +170,7 @@ jobs: uses: actions/cache@v4 with: path: ${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} - key: boost-tarball + key: ${{format('boost_tarball_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} - name: Download boost libraries if: steps.cache-boost-tarball.outputs.cache-hit != 'true' @@ -167,28 +181,69 @@ jobs: working-directory: ${{runner.temp}}/deps run: tar xf ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} - - uses: actions/checkout@v4 + - name: Checking out AWS SDK C++ source tree + uses: actions/checkout@v4 + with: + repository: aws/aws-sdk-cpp + ref: ${{format('{0}.{1}.{2}', env.AWS_SDK_CPP_MAJOR, env.AWS_SDK_CPP_MINOR, env.AWS_SDK_CPP_PATCH)}} + path: aws-sdk-cpp + submodules: recursive + + - name: Configure CMake for AWS SDK C++ + run: | + cmake \ + -B ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} \ + -S ${{github.workspace}}/aws-sdk-cpp \ + -DCMAKE_INSTALL_PREFIX=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \ + -DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \ + -DCMAKE_C_COMPILER=${{matrix.config.cc}} \ + -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \ + ${{matrix.config.aws_libcxx_cmake_flags}} \ + ${{matrix.config.aws_sanitizer_cmake_flags}} \ + -DCPP_STANDARD=20 \ + -DENABLE_UNITY_BUILD=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DFORCE_SHARED_CRT=OFF \ + -DENABLE_TESTING=OFF \ + -DAUTORUN_UNIT_TESTS=OFF \ + -DBUILD_ONLY=s3-crt + + - name: CMake info for AWS SDK C++ + run: cmake -L ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} + + - name: Build for AWS SDK C++ + run: cmake --build ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel + + - name: Install for AWS SDK C++ + run: cmake --install ${{github.workspace}}/../aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} + + - name: Checking out source tree + uses: actions/checkout@v4 + with: + fetch-depth: 2 - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type run: | cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ - -B ${{github.workspace}}/../build-${{matrix.config.label}} \ + -B ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} \ + -S ${{github.workspace}} \ -DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \ -DCMAKE_C_COMPILER=${{matrix.config.cc}} \ -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \ ${{matrix.config.libcxx_cmake_flags}} \ ${{matrix.config.sanitizer_cmake_flags}} \ + -DCMAKE_PREFIX_PATH=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \ -DBoost_ROOT=${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - name: CMake info - run: cmake -L ${{github.workspace}}/..//build-${{matrix.config.label}} + run: cmake -L ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} - name: Build # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/../build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel + run: cmake --build ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel - name: Info Clang Tidy if: matrix.config.run_clang_tidy @@ -197,7 +252,7 @@ jobs: - name: Clang Tidy if: matrix.config.run_clang_tidy # Run Clang Tidy - run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/../build-${{matrix.config.label}} + run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} - name: MTR tests if: matrix.config.run_mtr @@ -208,13 +263,13 @@ jobs: # Linking the "binlog_streaming" from the source tree into the MTR suits directory on the system sudo ln -s ${{github.workspace}}/mtr/binlog_streaming /usr/lib/mysql-test/suite/binlog_streaming # Running MTR from the system package - BINSRV=${{github.workspace}}/../build-${{matrix.config.label}}/binlog_server ./mtr \ + BINSRV=${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}}/binlog_server ./mtr \ --client-bindir=/usr/lib/mysql-test/bin --vardir=${{github.workspace}}/../mtrvardir \ --force --max-test-fail=0 --retry=0 --nounit-tests --big-test --repeat=2 --parallel=${{steps.cpu-cores.outputs.count}} \ --suite=binlog_streaming ${{matrix.config.mtr_options}} - name: CTest - working-directory: ${{github.workspace}}/../build-${{matrix.config.label}} + working-directory: ${{github.workspace}}/../percona-binlog-server-build-${{matrix.config.label}} # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C ${{matrix.config.build_type}} --parallel diff --git a/CMakeLists.txt b/CMakeLists.txt index 07fd8b2..8ed8a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,9 @@ find_package(Boost 1.83.0 EXACT REQUIRED) find_package(MySQL REQUIRED) +find_package(ZLIB REQUIRED) +find_package(AWSSDK REQUIRED COMPONENTS s3-crt) + set(source_files # main application files src/app.cpp @@ -167,6 +170,9 @@ set(source_files src/binsrv/main_config.hpp src/binsrv/main_config.cpp + src/binsrv/s3_storage_backend.hpp + src/binsrv/s3_storage_backend.cpp + src/binsrv/storage_fwd.hpp src/binsrv/storage.hpp src/binsrv/storage.cpp @@ -240,6 +246,7 @@ target_link_libraries(binlog_server binlog_server_compiler_flags PRIVATE Boost::headers MySQL::client + aws-cpp-sdk-s3-crt ) # for some reason it is not possible to propagate CXX_EXTENSIONS and # CXX_STANDARD_REQUIRED via interface library (binlog_server_compiler_flags) diff --git a/src/app.cpp b/src/app.cpp index 91d3e59..27662e0 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -208,6 +208,9 @@ int main(int argc, char *argv[]) { msg += ", path: "; msg += storage_config.get<"path">(); logger->log(binsrv::log_severity::info, msg); + msg = "description: "; + msg += storage_backend->get_description(); + logger->log(binsrv::log_severity::info, msg); binsrv::storage storage{std::move(storage_backend)}; logger->log(binsrv::log_severity::info, diff --git a/src/binsrv/basic_storage_backend.cpp b/src/binsrv/basic_storage_backend.cpp index 119f205..8790176 100644 --- a/src/binsrv/basic_storage_backend.cpp +++ b/src/binsrv/basic_storage_backend.cpp @@ -51,4 +51,8 @@ void basic_storage_backend::close_stream() { stream_opened_ = false; } +[[nodiscard]] std::string basic_storage_backend::get_description() const { + return do_get_description(); +} + } // namespace binsrv diff --git a/src/binsrv/basic_storage_backend.hpp b/src/binsrv/basic_storage_backend.hpp index fb1d41c..fdafcf1 100644 --- a/src/binsrv/basic_storage_backend.hpp +++ b/src/binsrv/basic_storage_backend.hpp @@ -28,6 +28,8 @@ class basic_storage_backend { void write_data_to_stream(util::const_byte_span data); void close_stream(); + [[nodiscard]] std::string get_description() const; + private: bool stream_opened_{false}; @@ -39,6 +41,8 @@ class basic_storage_backend { virtual void do_open_stream(std::string_view name) = 0; virtual void do_write_data_to_stream(util::const_byte_span data) = 0; virtual void do_close_stream() = 0; + + [[nodiscard]] virtual std::string do_get_description() const = 0; }; } // namespace binsrv diff --git a/src/binsrv/filesystem_storage_backend.cpp b/src/binsrv/filesystem_storage_backend.cpp index 9ab2f0c..ed50b52 100644 --- a/src/binsrv/filesystem_storage_backend.cpp +++ b/src/binsrv/filesystem_storage_backend.cpp @@ -117,6 +117,11 @@ void filesystem_storage_backend::do_write_data_to_stream( void filesystem_storage_backend::do_close_stream() { ofs_.close(); } +[[nodiscard]] std::string +filesystem_storage_backend::do_get_description() const { + return "local filesystem"; +} + [[nodiscard]] std::filesystem::path filesystem_storage_backend::get_object_path(std::string_view name) const { auto result{root_path_}; diff --git a/src/binsrv/filesystem_storage_backend.hpp b/src/binsrv/filesystem_storage_backend.hpp index 91d3189..9b8b6a5 100644 --- a/src/binsrv/filesystem_storage_backend.hpp +++ b/src/binsrv/filesystem_storage_backend.hpp @@ -31,6 +31,8 @@ class [[nodiscard]] filesystem_storage_backend : public basic_storage_backend { void do_write_data_to_stream(util::const_byte_span data) override; void do_close_stream() override; + [[nodiscard]] std::string do_get_description() const override; + [[nodiscard]] std::filesystem::path get_object_path(std::string_view name) const; }; diff --git a/src/binsrv/s3_storage_backend.cpp b/src/binsrv/s3_storage_backend.cpp new file mode 100644 index 0000000..b43e8b6 --- /dev/null +++ b/src/binsrv/s3_storage_backend.cpp @@ -0,0 +1,70 @@ +#include "binsrv/s3_storage_backend.hpp" + +#include +#include +#include +#include +#include + +#include + +// TODO: remove this include when switching to clang-18 where transitive +// IWYU 'export' pragmas are handled properly +#include "binsrv/basic_storage_backend_fwd.hpp" + +#include "util/byte_span.hpp" +#include "util/impl_helpers.hpp" + +namespace binsrv { + +using options_deimpl = util::deimpl; + +void s3_storage_backend::options_deleter::operator()(void *ptr) const noexcept { + auto *casted_ptr{static_cast(ptr)}; + Aws::ShutdownAPI(*casted_ptr); + // deleting via std::default_delete to avoid + // cppcoreguidelines-owning-memory warnings + using delete_helper = std::default_delete; + delete_helper{}(casted_ptr); +} + +s3_storage_backend::s3_storage_backend(std::string_view root_path) + : root_path_{root_path}, options_{new Aws::SDKOptions} { + Aws::InitAPI(*options_deimpl::get(options_)); +} + +[[nodiscard]] storage_object_name_container +s3_storage_backend::do_list_objects() { + storage_object_name_container result; + return result; +} + +[[nodiscard]] std::string +s3_storage_backend::do_get_object(std::string_view /*name*/) { + static constexpr std::size_t file_size{8}; + std::string file_content(file_size, 'x'); + return file_content; +} + +void s3_storage_backend::do_put_object(std::string_view /*name*/, + util::const_byte_span /*content*/) {} + +void s3_storage_backend::do_open_stream(std::string_view /*name*/) {} + +void s3_storage_backend::do_write_data_to_stream( + util::const_byte_span /*data*/) {} + +void s3_storage_backend::do_close_stream() {} + +[[nodiscard]] std::string s3_storage_backend::do_get_description() const { + std::string res{"AWS S3 (SDK "}; + const auto &casted_options = *options_deimpl::get(options_); + res += std::to_string(casted_options.sdkVersion.major); + res += '.'; + res += std::to_string(casted_options.sdkVersion.minor); + res += '.'; + res += std::to_string(casted_options.sdkVersion.patch); + res += ')'; + return res; +} +} // namespace binsrv diff --git a/src/binsrv/s3_storage_backend.hpp b/src/binsrv/s3_storage_backend.hpp new file mode 100644 index 0000000..03c0502 --- /dev/null +++ b/src/binsrv/s3_storage_backend.hpp @@ -0,0 +1,42 @@ +#ifndef BINSRV_S3_STORAGE_BACKEND_HPP +#define BINSRV_S3_STORAGE_BACKEND_HPP + +#include +#include + +#include "binsrv/basic_storage_backend.hpp" // IWYU pragma: export + +namespace binsrv { + +class [[nodiscard]] s3_storage_backend : public basic_storage_backend { +public: + explicit s3_storage_backend(std::string_view root_path); + + [[nodiscard]] const std::filesystem::path &get_root_path() const noexcept { + return root_path_; + } + +private: + std::filesystem::path root_path_; + struct options_deleter { + void operator()(void *ptr) const noexcept; + }; + using options_ptr = std::unique_ptr; + options_ptr options_; + + [[nodiscard]] storage_object_name_container do_list_objects() override; + + [[nodiscard]] std::string do_get_object(std::string_view name) override; + void do_put_object(std::string_view name, + util::const_byte_span content) override; + + void do_open_stream(std::string_view name) override; + void do_write_data_to_stream(util::const_byte_span data) override; + void do_close_stream() override; + + [[nodiscard]] std::string do_get_description() const override; +}; + +} // namespace binsrv + +#endif // BINSRV_S3_STORAGE_BACKEND_HPP diff --git a/src/binsrv/storage_backend_factory.cpp b/src/binsrv/storage_backend_factory.cpp index 94586d5..28e3a04 100644 --- a/src/binsrv/storage_backend_factory.cpp +++ b/src/binsrv/storage_backend_factory.cpp @@ -5,6 +5,7 @@ #include "binsrv/basic_storage_backend_fwd.hpp" #include "binsrv/filesystem_storage_backend.hpp" +#include "binsrv/s3_storage_backend.hpp" #include "binsrv/storage_config.hpp" #include "util/exception_location_helpers.hpp" @@ -14,11 +15,14 @@ namespace binsrv { basic_storage_backend_ptr storage_backend_factory::create(const storage_config &config) { const auto &storage_backend_type = config.get<"type">(); - if (storage_backend_type != "fs") { - util::exception_location().raise( - "unknown storage backend type"); + if (storage_backend_type == "fs") { + return std::make_unique(config.get<"path">()); } - return std::make_unique(config.get<"path">()); + if (storage_backend_type == "s3") { + return std::make_unique(config.get<"path">()); + } + util::exception_location().raise( + "unknown storage backend type"); } } // namespace binsrv