Fix loading a nested archive

The problem was that an intermediate archive failed to load because of the table validation failure

immer/extra/archive/champ/champ.hpp

@@ -37,9 +37,10 @@ struct node_traits
 class hash_validation_failed_exception : public archive_exception
 {
 public:
-    hash_validation_failed_exception()
+    explicit hash_validation_failed_exception(const std::string& msg)
         : archive_exception{"Hash validation failed, likely different hash "
-                            "algos are used for saving and loading"}
+                            "algos are used for saving and loading, " +
+                            msg}
     {
     }
 };
@@ -59,7 +60,7 @@ class container_loader
     {
         const value_t* operator()(const value_t& v) const noexcept
         {
-            return &v;
+            return std::addressof(v);
         }
     };
 
@@ -103,10 +104,13 @@ class container_loader
                     project_value_ptr,
                     immer::detail::constantly<const value_t*, nullptr>>(item);
                 if (!p) {
-                    throw hash_validation_failed_exception{};
+                    throw hash_validation_failed_exception{
+                        "Couldn't find an element"};
                 }
                 if (!(*p == item)) {
-                    throw hash_validation_failed_exception{};
+                    throw hash_validation_failed_exception{
+                        "Found element is not equal to the one we were looking "
+                        "for"};
                 }
             }
         }

immer/extra/archive/json/archivable.hpp

@@ -8,6 +8,8 @@
 
 #include <fmt/format.h>
 
+#include <boost/core/demangle.hpp>
+
 namespace immer::archive {
 
 namespace detail {
@@ -135,10 +137,13 @@ void load_minimal(
     try {
         value.container = loader.load(container_id_{id});
     } catch (const archive_exception& ex) {
-        throw ::cereal::Exception{
-            fmt::format("Failed to load a container ID {} from the archive: {}",
-                        id,
-                        ex.what())};
+        if (!ar.get_input_archives().ignore_archive_exceptions) {
+            throw ::cereal::Exception{fmt::format(
+                "Failed to load a container ID {} from the archive of {}: {}",
+                id,
+                boost::core::demangle(typeid(Container).name()),
+                ex.what())};
+        }
     }
 }
 

immer/extra/archive/json/json_immer.hpp

@@ -88,6 +88,7 @@ class json_immer_input_archive
     }
 
     ImmerArchives& get_input_archives() { return archives; }
+    const ImmerArchives& get_input_archives() const { return archives; }
 
 private:
     cereal::JSONInputArchive archive;

immer/extra/archive/json/json_with_archive.hpp

@@ -183,6 +183,7 @@ struct archives_load
     using names_t = Names;
 
     Storage storage;
+    bool ignore_archive_exceptions = false;
 
     template <class Container>
     auto& get_loader()
@@ -541,6 +542,9 @@ auto load_archives(std::istream& is,
     auto prev = archives;
     while (true) {
         // Keep reloading until everything is loaded.
+        // Reloading of the archive might trigger validation of some containers
+        // (hash-based, for example) because the elements actually come from
+        // other archives that are not yet loaded.
         archives = reload_archive(is, std::move(archives));
         if (prev == archives) {
             break;
@@ -552,8 +556,9 @@ auto load_archives(std::istream& is,
 }
 
 constexpr auto reload_archive = [](std::istream& is, auto archives) {
-    using Archives = std::decay_t<decltype(archives)>;
-    auto restore   = util::istream_snapshot{is};
+    using Archives                     = std::decay_t<decltype(archives)>;
+    auto restore                       = util::istream_snapshot{is};
+    archives.ignore_archive_exceptions = true;
     auto ar = json_immer_input_archive<Archives>{std::move(archives), is};
     /**
      * NOTE: Critical to clear the archives before loading into it

immer/extra/archive/json/json_with_archive_auto.hpp

@@ -229,9 +229,10 @@ auto load_initial_auto_archives(std::istream& is, WrapF wrap)
 
 constexpr auto reload_archive_auto = [](auto wrap) {
     return [wrap](std::istream& is, auto archives) {
-        using Archives = std::decay_t<decltype(archives)>;
-        using WrapF    = std::decay_t<decltype(wrap)>;
-        auto restore   = util::istream_snapshot{is};
+        using Archives                     = std::decay_t<decltype(archives)>;
+        using WrapF                        = std::decay_t<decltype(wrap)>;
+        auto restore                       = util::istream_snapshot{is};
+        archives.ignore_archive_exceptions = true;
         auto previous =
             json_immer_input_archive<Archives>{std::move(archives), is};
         auto ar = json_immer_auto_input_archive<decltype(previous), WrapF>{

test/extra/archive/CMakeLists.txt

@@ -20,6 +20,8 @@ add_executable(
   test_conversion.cpp
   test_circular_dependency_conversion.cpp
   ${PROJECT_SOURCE_DIR}/immer/extra/archive/xxhash/xxhash_64.cpp)
+target_precompile_headers(archive-tests PRIVATE
+                          <immer/extra/archive/json/json_with_archive_auto.hpp>)
 add_dependencies(tests archive-tests)
 add_test("test/archive-tests" archive-tests)
 target_include_directories(archive-tests PRIVATE ${CMAKE_SOURCE_DIR})

test/extra/archive/test_circular_dependency_conversion.cpp

@@ -82,8 +82,21 @@ struct value_two
     int number                 = {};
     vector_one<value_one> ones = {};
     key key                    = {};
+
+    friend std::ostream& operator<<(std::ostream& s, const value_two& value)
+    {
+        return s << fmt::format("number = {}, ones = {}, key = '{}'",
+                                value.number,
+                                value.ones.size(),
+                                value.key.str);
+    }
 };
 
+std::ostream& operator<<(std::ostream& s, const two_boxed& value)
+{
+    return s << value.two.get();
+}
+
 const key& get_table_key(const two_boxed& two) { return two.two.get().key; }
 
 std::size_t xx_hash_value(const two_boxed& value)
@@ -98,6 +111,10 @@ two_boxed::two_boxed(value_two val)
 
 } // namespace model
 
+template <>
+struct fmt::formatter<model::two_boxed> : ostream_formatter
+{};
+
 BOOST_HANA_ADAPT_STRUCT(model::value_two, number, ones, key);
 
 namespace model {
@@ -224,6 +241,14 @@ TEST_CASE("Test exception while circular converting")
         immer::archive::to_json_with_auto_archive(value, names);
     // REQUIRE(json_str == "");
 
+    SECTION("Try to load")
+    {
+        const auto loaded =
+            immer::archive::from_json_with_auto_archive<model::value_one>(
+                json_str, names);
+        REQUIRE(loaded == value);
+    }
+
     /**
      * NOTE: There is a circular dependency between archives: to convert
      * value_one we need to convert value_two and vice versa.
@@ -554,16 +579,6 @@ TEST_CASE("Test circular dependency archives", "[conversion]")
             REQUIRE(format_twos.impl().root == format_twos_2.impl().root);
         }
         REQUIRE(test::to_json(value.twos_table) == test::to_json(format_twos));
-
-        // SECTION("Compare structure")
-        // {
-        //     const auto [format_twos_json, ar] =
-        //         immer::archive::to_json_with_auto_archive(format_twos,
-        //                                                   format_names);
-        //     const auto [model_twos_json, ar2] =
-        //         immer::archive::to_json_with_auto_archive(value, names);
-        //     REQUIRE(model_twos_json == format_twos_json);
-        // }
     }
 
     SECTION("map")
@@ -635,4 +650,23 @@ TEST_CASE("Test circular dependency archives", "[conversion]")
             REQUIRE(model_twos_json == format_twos_json);
         }
     }
+
+    SECTION("everything")
+    {
+        const auto convert = [&](const auto& value) {
+            return immer::archive::convert_container(
+                model_archives, format_load_archives, value);
+        };
+        const auto format_value = [&] {
+            auto result = format::value_one{};
+            hana::for_each(hana::keys(result), [&](auto key) {
+                hana::at_key(result, key) = convert(hana::at_key(value, key));
+            });
+            return result;
+        }();
+        const auto [format_json_str, model_archives] =
+            immer::archive::to_json_with_auto_archive(format_value,
+                                                      format_names);
+        REQUIRE(format_json_str == json_str);
+    }
 }

test/extra/archive/test_special_archive.cpp

@@ -1,5 +1,6 @@
 #include <catch2/catch_test_macros.hpp>
 #include <catch2/matchers/catch_matchers_exception.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
 
 #include "utils.hpp"
 
@@ -574,8 +575,11 @@ TEST_CASE("Special archive throws cereal::Exception")
     REQUIRE_THROWS_MATCHES(
         immer::archive::from_json_with_archive<test_data>(json_archive_str),
         ::cereal::Exception,
-        Catch::Matchers::Message("Failed to load a container ID 99 from the "
-                                 "archive: Container ID 99 is not found"));
+        MessageMatches(Catch::Matchers::ContainsSubstring(
+                           "Failed to load a container ID 99 from the "
+                           "archive of immer::vector<int") &&
+                       Catch::Matchers::ContainsSubstring(
+                           "Container ID 99 is not found")));
 }
 
 namespace {