[Feat] Overlaybd-streamConv:

   - support parsing a YAML file to configure server
   - docker image support

Signed-off-by: Yifan Yuan <[email protected]>

,

CMakeLists.txt

@@ -42,6 +42,16 @@ set(ENABLE_MIMIC_VDSO off)
 
 find_package(photon REQUIRED)
 find_package(tcmu REQUIRED)
+find_package(yamlcpp)
+if (NOT yamlcpp_FOUND)
+  FetchContent_Declare(
+    yamlcpp
+    GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
+    GIT_TAG 0.8.0
+  )
+  FetchContent_MakeAvailable(yamlcpp)
+endif()
+
 
 if(BUILD_TESTING)
   enable_testing()

src/example_config/stream-conv.yaml

@@ -0,0 +1,19 @@
+# // {
+# //     "serverConfig": {
+# //         "uds":  "/var/run/stream_conv.sock",
+# //         "workdir": "/tmp/stream_conv"
+# //     },
+# //     "logConfig": {
+# //         "level": 0,
+# //         "mode": "stdout"
+# //     }
+# // }
+globalConfig:
+  servAddr: /var/run/stream_conv.sock
+  workDir: /tmp/stream_conv
+  logConfig:
+    level: 0
+    mode: stdout
+    rotateNum: 3
+    limitSizeMB: 10
+    path: /var/log/overlaybd/stream_convertor.log

src/overlaybd/gzip/gz.cpp

@@ -26,6 +26,7 @@
 #include <photon/net/socket.h>
 #include "../../tools/sha256file.h"
 #include "../gzindex/gzfile_index.h"
+#include "photon/common/alog.h"
 class GzAdaptorFile : public photon::fs::VirtualReadOnlyFile {
 public:
     GzAdaptorFile() {
@@ -133,6 +134,9 @@ class GzStreamFile :  public IGzFile {
                     len = sizeof(buf);
                 }
                 auto readn = this->read(buf, len);
+                if (readn <= 0){
+                    LOG_ERRNO_RETURN(EIO, -1, "read buffer error");
+                }
                 offset -= readn;
             }
             return cur_offset;
@@ -166,9 +170,14 @@ class GzStreamFile :  public IGzFile {
             if (strm.avail_in < 0) {
                 LOG_ERRNO_RETURN(0, -1, "read buffer from uds failed");
             }
-            size_t readn = strm.avail_in;
             if (strm.avail_in == 0)
                 break;
+            if (!check_type) {
+                if (!((uint8_t)in[0] == 0x1f && (uint8_t)in[1] == 0x8b)){
+                    LOG_ERRNO_RETURN(EIO, -1, "buffer is not gzip type");
+                }
+                check_type = true;
+            }
             LOG_DEBUG("recv: `", strm.avail_in);
             st_size += strm.avail_in;
             strm.next_in = in;
@@ -243,6 +252,7 @@ class GzStreamFile :  public IGzFile {
     ssize_t st_size = 0;
     IFile *m_file = nullptr;
     photon::fs::IFileSystem *m_fs = nullptr;
+    bool check_type = false;
 
     const static int CHUNK = 32768;
     IStream *fstream;

src/overlaybd/stream_convertor/CMakeLists.txt

@@ -1,14 +1,16 @@
 file(GLOB SOURCE_SERV "*.cpp")
 
-add_executable(stream_conv ${SOURCE_SERV})
-target_include_directories(stream_conv PUBLIC
+add_executable(overlaybd-streamConv ${SOURCE_SERV})
+target_include_directories(overlaybd-streamConv PUBLIC
     ${PHOTON_INCLUDE_DIR}
+    ${rapidjson_SOURCE_DIR}/include
 )
-target_link_libraries(stream_conv
-photon_static
-gzip_lib
-gzindex_lib
-tar_lib
+target_link_libraries(overlaybd-streamConv
+    photon_static
+    gzip_lib
+    gzindex_lib
+    tar_lib
+    yaml-cpp
 )
 
 # if(BUILD_TESTING)

src/overlaybd/stream_convertor/config.h

@@ -0,0 +1,52 @@
+/*
+   Copyright The Overlaybd Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+#pragma once
+
+#include <string>
+#include <vector>
+#include "config_utils.h"
+
+namespace App {
+
+struct LogConfigPara : public App::ConfigGroup {
+    APPCFG_CLASS
+    APPCFG_PARA(level, uint32_t, 1);
+    APPCFG_PARA(path, std::string, "/var/log/overlaybd/stream-convertor.log");
+    APPCFG_PARA(limitSizeMB, uint32_t, 10);
+    APPCFG_PARA(rotateNum, int, 3);
+    APPCFG_PARA(mode, std::string, "stdout");
+};
+
+// struct ServerConfigPara : public App::ConfigGroup  {
+//     APPCFG_CLASS
+
+// };
+
+struct GlobalConfigPara : public App::ConfigGroup  {
+    APPCFG_CLASS;
+    APPCFG_PARA(servAddr, std::string, "/var/run/stream_conv.sock");
+    APPCFG_PARA(workDir, std::string, "/tmp/stream_conv");
+    //APPCFG_PARA(ServerConfig, ServerConfigPara);
+    APPCFG_PARA(logConfig, LogConfigPara);
+};
+
+struct AppConfig : public App::ConfigGroup {
+    APPCFG_CLASS
+
+    APPCFG_PARA(globalConfig, GlobalConfigPara);
+};
+
+} // namespace ImageConfigNS

src/overlaybd/stream_convertor/config_utils.h

@@ -0,0 +1,84 @@
+#pragma once
+#include <photon/common/utility.h>
+#include <yaml-cpp/yaml.h>
+#include <iostream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace App {
+
+struct ConfigGroup : public YAML::Node {
+    ConfigGroup() = default;
+    ConfigGroup(const YAML::Node &node) : YAML::Node(node){};
+    ConfigGroup(const std::string &filename) {
+        parseYAML(std::move(filename));
+    }
+
+    void parseYAML(const std::string &fn) {
+        Clone(YAML::LoadFile(fn));
+    }
+
+    static size_t charfilter(char *dst, const char *src, char extract, size_t maxlen = 256UL) {
+        size_t i;
+        for (i = 0; (*src) && i < maxlen; i++, src++) {
+            while (*src == '-') {
+                src++;
+            }
+            if (!(*src))
+                break;
+            dst[i] = *(src);
+        }
+        dst[i] = 0;
+        return i;
+    }
+};
+
+#define APPCFG_PARA(ParaName, ParaType, ...)                                                       \
+    inline ParaType ParaName() const {                                                             \
+        return operator[](#ParaName).as<ParaType>(__VA_ARGS__);                                    \
+    }
+
+#define APPCFG_CLASS using ConfigGroup::ConfigGroup;
+
+// merge two yaml nodes
+// generate new node with full data
+static YAML::Node mergeConfig(const YAML::Node &lhs, const YAML::Node &rhs) {
+    // if one of nodes is not type of map
+    // just return rhs
+    if (lhs.Type() != YAML::NodeType::Map || rhs.Type() != YAML::NodeType::Map)
+        return YAML::Clone(rhs);
+    // both are map, merge two maps
+    YAML::Node ret = YAML::Clone(lhs);
+    for (auto &node : rhs) {
+        auto key = node.first.as<std::string>();
+        if (ret[key].IsDefined()) {
+            // if key exists in lhs, merge recursivily
+            ret[key] = mergeConfig(lhs[key], node.second);
+        } else {
+            // just add as new key
+            ret[key] = Clone(node.second);
+        }
+    }
+    return ret;
+}
+
+} // namespace App
+
+namespace YAML {
+
+template <typename T>
+struct convert {
+    template <ENABLE_IF_BASE_OF(App::ConfigGroup, T)>
+    static Node encode(const T &rhs) {
+        return rhs;
+    }
+
+    template <ENABLE_IF_BASE_OF(App::ConfigGroup, T)>
+    static bool decode(const Node &node, T &rhs) {
+        rhs = T(node);
+        return true;
+    }
+};
+
+} // namespace YAML