GMS Tool refactor (#2)

GMS Tool: 
 * Load from ZIP
 * Export uncompressed GMS
 * Export decompiled LOC
LOCC:
 * Created compiler & decompiler
* Added tests

.gitignore

@@ -2,4 +2,7 @@
 .vscode
 build
 cmake-build-*
-venv
+venv
+
+# Ignore zlib generated header
+/Modules/zlib/zconf.h.included

.gitmodules

@@ -7,3 +7,12 @@
 [submodule "Modules/zlib"]
 	path = Modules/zlib
 	url = https://github.com/madler/zlib
+[submodule "Modules/spdlog"]
+	path = Modules/spdlog
+	url = https://github.com/gabime/spdlog
+[submodule "Modules/gtest"]
+	path = Modules/gtest
+	url = https://github.com/google/googletest
+[submodule "Modules/CLI11"]
+	path = Modules/CLI11
+	url = https://github.com/CLIUtils/CLI11.git

CMakeLists.txt

@@ -16,11 +16,20 @@ SET(JSON_Install OFF CACHE BOOL "Do not generate install target in nlohmann")
 SET(JSON_BuildTests OFF CACHE BOOL "Do not build tests of nlohmann")
 SET(FMT_INSTALL OFF CACHE BOOL "Do not generate install targets of fmt")
 SET(FMT_TEST OFF CACHE BOOL "Do not generate install targets of fmt")
+SET(ZLIB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/Modules/zlib")
+SET(SPDLOG_FMT_EXTERNAL_HO ON CACHE BOOL "Do not use own fmt")
+SET(BUILD_SHARED_LIBS OFF)
 
 # Modules
 add_subdirectory(Modules/fmt)
 add_subdirectory(Modules/zlib)
+add_subdirectory(Modules/CLI11)
+add_subdirectory(Modules/gtest)
+add_subdirectory(Modules/spdlog)
+add_subdirectory(Modules/minizip)
 add_subdirectory(Modules/nlohmann)
+add_subdirectory(Modules/BMLOC)
 
 # Our projects
-add_subdirectory(Tools/GMSInfo)
+add_subdirectory(Tools/GMSInfo)
+add_subdirectory(Tools/LOCC)

Modules/BMLOC/CMakeLists.txt

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.16)
+project(BMLOC)
+
+set(CMAKE_CXX_STANDARD 20)
+
+file(GLOB_RECURSE BMLOC_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/source/*.cpp)
+add_library(BMLOC STATIC ${BMLOC_SOURCES})
+add_library(BMFormats::Localization ALIAS BMLOC)
+
+target_compile_definitions(BMLOC PRIVATE -D_CRT_SECURE_NO_WARNINGS=1)
+target_include_directories(BMLOC PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(BMLOC PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/internal/include)
+target_link_libraries(BMLOC PUBLIC nlohmann_json)
+
+# Tests
+enable_testing()
+
+file(GLOB_RECURSE BLOC_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cpp)
+
+add_executable(BMLOC_Tests ${BLOC_TEST_SOURCES})
+target_link_libraries(BMLOC_Tests BMFormats::Localization gtest gmock gtest_main)
+
+add_test(BMLOC_AllTests BMLOC_Tests)

Modules/BMLOC/README.md

@@ -0,0 +1,4 @@
+BMLOC
+------
+
+ Blood Money LOC file support

Modules/BMLOC/include/BM/LOC/LOCJson.h

@@ -0,0 +1,195 @@
+#pragma once
+
+
+#include <nlohmann/json.hpp>
+
+#include <BM/LOC/LOCTypes.h>
+#include <BM/LOC/LOCTree.h>
+
+namespace nlohmann
+{
+    template <>
+    struct adl_serializer<BM::LOC::HBM_MissionObjectiveType>
+    {
+        static void to_json(json& j, const BM::LOC::HBM_MissionObjectiveType& type)
+        {
+            switch (type)
+            {
+                case BM::LOC::HBM_Target:
+                case BM::LOC::HBM_Retrieve:
+                case BM::LOC::HBM_Escape:
+                case BM::LOC::HBM_Dispose:
+                case BM::LOC::HBM_Protect:
+                case BM::LOC::HBM_Optional:
+                    j = static_cast<char>(type);
+                    break;
+                default:
+                    j = static_cast<uint8_t>(type);
+                    break;
+            }
+        }
+
+        static void from_json(const json& j, BM::LOC::HBM_MissionObjectiveType& r)
+        {
+            if (j.is_string())
+            {
+                r = static_cast<BM::LOC::HBM_MissionObjectiveType>(j.get<std::string>()[0]);
+            }
+            else
+            {
+                r = static_cast<BM::LOC::HBM_MissionObjectiveType>(j.get<uint8_t>());
+            }
+        }
+    };
+
+    template <>
+    struct adl_serializer<BM::LOC::TreeNodeType>
+    {
+        static constexpr const char* kValOrData = "data";
+        static constexpr const char* kNode = "node";
+
+        static void to_json(json& j, const BM::LOC::TreeNodeType& type)
+        {
+            switch (type)
+            {
+                case BM::LOC::TreeNodeType::VALUE_OR_DATA: j = kValOrData; break;
+                case BM::LOC::TreeNodeType::NODE_WITH_CHILDREN: j = kNode; break;
+                default: j = static_cast<int>(type); break;
+            }
+        }
+
+        static void from_json(const json& j, BM::LOC::TreeNodeType& r)
+        {
+            if (j == kValOrData) r = BM::LOC::TreeNodeType::VALUE_OR_DATA;
+            else if (j == kNode) r = BM::LOC::TreeNodeType::NODE_WITH_CHILDREN;
+            else r = static_cast<BM::LOC::TreeNodeType>(j.get<int>());
+        }
+    };
+
+    template <>
+    struct adl_serializer<BM::LOC::LOCTreeNode>
+    {
+        static constexpr const char* kNameToken = "name";
+        static constexpr const char* kValueToken = "value";
+        static constexpr const char* kTypeToken = "type";
+        static constexpr const char* kNumChildrenToken = "numChildren";
+        static constexpr const char* kChildrenListToken = "children";
+        static constexpr const char* kOriginalTypeByteToken = "org_tbyte";
+
+        static void to_json(json& j, const BM::LOC::LOCTreeNode* node)
+        {
+            adl_serializer<BM::LOC::TreeNodeType>::to_json(j[kTypeToken], node->nodeType);
+
+            switch (node->nodeType)
+            {
+                case BM::LOC::VALUE_OR_DATA:
+                    j[kNameToken] = node->name;
+
+                    if (!node->value.empty())
+                    {
+                        // If not string rewrite to byte export (calc length between next node in this level or in parent' level)
+                        std::string_view value { node->value };
+                        j[kValueToken] = value;
+                    }
+
+                    if (node->originalTypeRawData.has_value()) // our value was overridden, need to save original byte
+                    {
+                        j[kOriginalTypeByteToken] = node->originalTypeRawData.value();
+                    }
+                    break;
+                case BM::LOC::NODE_WITH_CHILDREN:
+                    if (!node->IsRoot()) { j[kNameToken] = node->name; }
+                    assert(node->numChild == node->children.size());
+
+                    j[kNumChildrenToken] = node->numChild;
+
+                    for (const auto& child : node->children)
+                    {
+                        json jc;
+                        nlohmann::adl_serializer<BM::LOC::LOCTreeNode>::to_json(jc, child);
+                        j[kChildrenListToken].push_back(jc);
+                    }
+                    break;
+            }
+        }
+
+        static void from_json(const json& j, BM::LOC::LOCTreeNode* node)
+        {
+            node->currentBufferPtr = nullptr;
+
+            if (auto nameIterator = j.find(kNameToken); nameIterator != j.end())
+            {
+                node->name = nameIterator->get<std::string>();
+            }
+            else if (!node->IsRoot())
+            {
+                throw std::exception { "Not allowed to store non-root anonymous node!" };
+            }
+
+            if (auto nodeTypeIter = j.find(kTypeToken); nodeTypeIter != j.end())
+            {
+                nlohmann::adl_serializer<BM::LOC::TreeNodeType>::from_json(j[kTypeToken], node->nodeType);
+            }
+            else
+            {
+                throw std::exception { "Each node should contain type!" };
+            }
+
+            switch (node->nodeType)
+            {
+                case BM::LOC::VALUE_OR_DATA:
+                    if (auto valueIterator = j.find(kValueToken); valueIterator != j.end())
+                    {
+                        node->value = valueIterator->get<std::string>();
+                    } else throw std::exception { "Key 'value' not found for VALUE node!" };
+
+                    if (auto originalByteIterator = j.find(kOriginalTypeByteToken); originalByteIterator != j.end())
+                    {
+                        node->originalTypeRawData = originalByteIterator->get<uint8_t>();
+                    }
+                break;
+                case BM::LOC::NODE_WITH_CHILDREN:
+                {
+                    if (auto numChildIterator = j.find(kNumChildrenToken); numChildIterator != j.end())
+                    {
+                        node->numChild = numChildIterator->get<size_t>();
+                        node->children.reserve(node->numChild);
+                    }
+                    else throw std::exception { "Key 'numChild' not found for NWC node!" };
+
+                    if (auto childrenIterator = j.find(kChildrenListToken); childrenIterator != j.end())
+                    {
+                        if (!childrenIterator->is_array())
+                        {
+                            throw std::exception { "Key 'children' not array for NWC node!" };
+                        }
+
+                        const int requiredNumChild = node->numChild;
+
+                        for (int i = 0; i < requiredNumChild; i++)
+                        {
+                            auto child = new BM::LOC::LOCTreeNode(node, nullptr);
+
+                            try
+                            {
+                                auto jsonChild = childrenIterator->at(i);
+                                nlohmann::adl_serializer<BM::LOC::LOCTreeNode>::from_json(jsonChild, child);
+                                node->AddChild(child);
+                            }
+                            catch (const nlohmann::json::out_of_range& outOfRange)
+                            {
+                                throw std::exception { "Bad 'numChild' value in json for NWC node!" };
+                            }
+                        }
+                    } //Allowed to have 0 child nodes
+                }
+                break;
+                default:
+                {
+                    throw std::exception { "Unknown node type!" };
+                }
+                break;
+            }
+        }
+    };
+}

Modules/BMLOC/include/BM/LOC/LOCSupportMode.h

@@ -0,0 +1,14 @@
+#pragma once
+
+namespace BM::LOC
+{
+    enum class LOCSupportMode
+    {
+        Hitman_BloodMoney = 0x0004,
+        Hitman_Contracts  = 0x0003,
+        Hitman_2SA        = 0x0002,
+        Hitman_A47        = 0x0001,
+        // Generic mode
+        Generic = Hitman_BloodMoney
+    };
+}

Modules/BMLOC/include/BM/LOC/LOCTree.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <unordered_map>
+#include <string_view>
+#include <string>
+#include <vector>
+#include <memory>
+#include <optional>
+
+#include <cstdint>
+
+#include <BM/LOC/LOCTypes.h>
+#include <BM/LOC/LOCSupportMode.h>
+
+namespace BM::LOC
+{
+    struct LOCTreeNode
+    {
+        // Editor defs
+        /**
+         * @struct MemoryMarkup
+         * @brief In-memory location of the node. This value used only in the compiler!
+         */
+        struct MemoryMarkup
+        {
+            uint32_t StartsAt {0 }; ///< Node starts at
+            uint32_t EndsAt { 0 }; ///< Node ends at
+
+            MemoryMarkup() = default;
+            MemoryMarkup(uint32_t o, uint32_t e) : StartsAt(o), EndsAt(e) {}
+        };
+
+        // Defs
+        static constexpr const char* kNoName = "<NONAME>";
+
+        // Base info
+        LOCTreeNode* parent {nullptr}; //Pointer to parent node
+        char* currentBufferPtr {nullptr}; //Available only in decompiler (be aware, if original buffer deallocated this pointer will be broken)
+        std::string name{kNoName}; //Name of node (could be empty for ROOT node)
+        std::string value; //Always string? Check it later
+        TreeNodeType nodeType{0}; //See TreeNodeType for details
+        std::optional<uint8_t> originalTypeRawData; //Original raw data if it was overridden by decompiler (just for reconstruction)
+        std::optional<MemoryMarkup> memoryMarkup; //Only for compiler for fast memory position search
+
+        // Tree data
+        size_t numChild {0}; //Number of children nodes
+        std::vector<LOCTreeNode*> children {}; //List of children nodes (please, add node through AddChild!)
+
+        // Methods
+        LOCTreeNode(LOCTreeNode* p, char* b);
+        ~LOCTreeNode();
+        void AddChild(LOCTreeNode* node);
+        void RemoveChild(LOCTreeNode* node);
+        [[nodiscard]] bool IsRoot() const;
+        [[nodiscard]] bool IsEmpty() const;
+        [[nodiscard]] bool IsData() const;
+        [[nodiscard]] bool IsContainer() const;
+
+        // Parser
+        static LOCTreeNode* ReadFromMemory(char* buffer, size_t bufferSize, LOCSupportMode supportMode = LOCSupportMode::Generic);
+
+        // Compiler
+        using CacheDataBase = std::unordered_map<std::string, std::string>;
+        static void GenerateCacheDataBase(LOCTreeNode* root, CacheDataBase& cache);
+        static bool Compile(LOCTreeNode* root, std::vector<uint8_t>& compiledBuffer);
+
+        // Serializer
+        static void CompileAndSave(LOCTreeNode* root, std::string_view pathToFile);
+
+        // Comparator
+        static bool Compare(LOCTreeNode* a, LOCTreeNode* b);
+
+    private:
+        /**
+         * @fn SortKeys
+         * @brief Should be called after AddChild or RemoveChild!
+         */
+        void SortKeys();
+    };
+}

Modules/BMLOC/include/BM/LOC/LOCTreeCompiler.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <BM/LOC/LOCTree.h>
+#include <BM/LOC/LOCSupportMode.h>
+
+#include <vector>
+
+#include <cstdint>
+
+namespace BM::LOC
+{
+    class LOCTreeCompiler
+    {
+    public:
+        using Buffer = std::vector<uint8_t>;
+
+        static bool Compile(Buffer& buffer, LOCTreeNode* rootNode, LOCSupportMode supportMode = LOCSupportMode::Generic);
+        static void MarkupTree(LOCTreeNode* rootNode);
+    };
+}