From 519faf027405a681669220fe2f0256a0f12569f4 Mon Sep 17 00:00:00 2001 From: Webster Sheets Date: Thu, 7 Dec 2023 02:51:59 -0500 Subject: [PATCH 1/4] Serializer: add RegisterPersistent - Inspired by Eris/Pluto's concept of "persistent objects" which have all references transparently swapped for the runtime-current instance of that value - Since a common pattern in Lua is to use the version of the value from the savefile if it's no longer defined in code, this implementation requires that persistent objects are also serializable --- data/meta/Serializer.lua | 7 ++ src/Game.cpp | 2 + src/lua/LuaSerializer.cpp | 191 ++++++++++++++++++++++++++++---------- src/lua/LuaSerializer.h | 4 + 4 files changed, 154 insertions(+), 50 deletions(-) diff --git a/data/meta/Serializer.lua b/data/meta/Serializer.lua index 4841e7d04f1..bb1afe51609 100644 --- a/data/meta/Serializer.lua +++ b/data/meta/Serializer.lua @@ -25,4 +25,11 @@ function Serializer:Register(key, serialize, unserialize) end ---@param class table function Serializer:RegisterClass(key, class) end +-- Register a table as a "persistent" value. All references to the saved +-- instance of that table will be transparently replaced across savegames to +-- maintain table instance identity. +---@param key string +---@param value table +function Serializer:RegisterPersistent(key, value) end + return Serializer diff --git a/src/Game.cpp b/src/Game.cpp index bec95e50ecc..0a6d6bbbc50 100644 --- a/src/Game.cpp +++ b/src/Game.cpp @@ -159,6 +159,7 @@ Game::Game(const Json &jsonObj) : // Preparing the Lua stuff Pi::luaSerializer->InitTableRefs(); + Pi::luaSerializer->LoadPersistent(jsonObj); GalacticEconomy::LoadFromJson(jsonObj); @@ -214,6 +215,7 @@ void Game::ToJson(Json &jsonObj) PROFILE_SCOPED() // preparing the lua serializer Pi::luaSerializer->InitTableRefs(); + Pi::luaSerializer->SavePersistent(jsonObj); // version jsonObj["version"] = s_saveVersion; diff --git a/src/lua/LuaSerializer.cpp b/src/lua/LuaSerializer.cpp index 9573e0694c8..6da666fbd6e 100644 --- a/src/lua/LuaSerializer.cpp +++ b/src/lua/LuaSerializer.cpp @@ -13,6 +13,13 @@ #include "core/Log.h" #include "profiler/Profiler.h" +// Well-known names of various serialization-related caches stored in the +// Lua Registry +static const char *NS_REFTABLE = "PiSerializerTableRefs"; +static const char *NS_CLASSES = "PiSerializerClasses"; +static const char *NS_CALLBACKS = "PiSerializerCallbacks"; +static const char *NS_PERSISTENT = "PiSerializerPersistent"; + // every module can save one object. that will usually be a table. we call // each serializer in turn and capture its return value we build a table like // so: @@ -81,7 +88,7 @@ void LuaSerializer::pickle_json(lua_State *l, int to_serialize, Json &out, const out["lua_class"] = cl; - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerClasses"); + lua_getfield(l, LUA_REGISTRYINDEX, NS_CLASSES); lua_getfield(l, -1, cl); if (lua_isnil(l, -1)) @@ -132,9 +139,9 @@ void LuaSerializer::pickle_json(lua_State *l, int to_serialize, Json &out, const lua_Integer ptr = lua_Integer(lua_topointer(l, to_serialize)); lua_pushinteger(l, ptr); // ptr - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); // ptr reftable - lua_pushvalue(l, -2); // ptr reftable ptr - lua_rawget(l, -2); // ptr reftable ??? + lua_getfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); // ptr reftable + lua_pushvalue(l, -2); // ptr reftable ptr + lua_rawget(l, -2); // ptr reftable ??? out["ref"] = ptr; @@ -259,11 +266,11 @@ void LuaSerializer::unpickle_json(lua_State *l, const Json &value) if (value.count("table")) { lua_newtable(l); - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); // [t] [refs] - lua_pushinteger(l, ptr); // [t] [refs] [key] - lua_pushvalue(l, -3); // [t] [refs] [key] [t] - lua_rawset(l, -3); // [t] [refs] - lua_pop(l, 1); // [t] + lua_getfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); // [t] [refs] + lua_pushinteger(l, ptr); // [t] [refs] [key] + lua_pushvalue(l, -3); // [t] [refs] [key] [t] + lua_rawset(l, -3); // [t] [refs] + lua_pop(l, 1); // [t] const Json &inner = value["table"]; if (inner.size() % 2 != 0) { @@ -278,9 +285,9 @@ void LuaSerializer::unpickle_json(lua_State *l, const Json &value) LUA_DEBUG_CHECK(l, 1); } else { // Reference to a previously-pickled table. - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); // [refs] - lua_pushinteger(l, ptr); // [refs] [key] - lua_rawget(l, -2); // [refs] [out] + lua_getfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); // [refs] + lua_pushinteger(l, ptr); // [refs] [key] + lua_rawget(l, -2); // [refs] [out] if (lua_isnil(l, -1)) throw SavedGameCorruptException(); @@ -294,7 +301,7 @@ void LuaSerializer::unpickle_json(lua_State *l, const Json &value) const char *cl = value["lua_class"].get_ref().c_str(); // If this was a full definition (not just a reference) then run the class's unserialiser function. if (value.count("table")) { - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerClasses"); + lua_getfield(l, LUA_REGISTRYINDEX, NS_CLASSES); lua_pushstring(l, cl); lua_gettable(l, -2); lua_remove(l, -2); @@ -317,11 +324,11 @@ void LuaSerializer::unpickle_json(lua_State *l, const Json &value) // Update the TableRefs cache with the new value // NOTE: recursive references to the original table will not be affected, // only references in tables deserialized later. - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); // [t] [refs] - lua_pushinteger(l, ptr); // [t] [refs] [key] - lua_pushvalue(l, -3); // [t] [refs] [key] [t] - lua_rawset(l, -3); // [t] [refs] - lua_pop(l, 1); // [t] + lua_getfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); // [t] [refs] + lua_pushinteger(l, ptr); // [t] [refs] [key] + lua_pushvalue(l, -3); // [t] [refs] [key] [t] + lua_rawset(l, -3); // [t] [refs] + lua_pop(l, 1); // [t] } } } @@ -345,8 +352,9 @@ void LuaSerializer::InitTableRefs() lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializer"); lua_newtable(l); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); + lua_setfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); + // NOTE: this is depended on by LuaRef.cpp lua_newtable(l); lua_setfield(l, LUA_REGISTRYINDEX, "PiLuaRefLoadTable"); } @@ -359,12 +367,85 @@ void LuaSerializer::UninitTableRefs() lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializer"); lua_pushnil(l); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerTableRefs"); + lua_setfield(l, LUA_REGISTRYINDEX, NS_REFTABLE); lua_pushnil(l); lua_setfield(l, LUA_REGISTRYINDEX, "PiLuaRefLoadTable"); } +void LuaSerializer::SavePersistent(Json &json) +{ + lua_State *l = Lua::manager->GetLuaState(); + LUA_DEBUG_START(l); + + // NOTE: this must be an array for consistent deserialization order + Json persist = Json::array(); + + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_PERSISTENT); + lua_pushnil(l); + + while (lua_next(l, -2)) { + Json entry = Json::object(); + + // persistent, id, value + std::string id = LuaPull(l, -2); + + // Serialize all registered persistent objects as a "backup" version + // (This will populate the PiSerializerTableRefs cache) + entry["persistId"] = id; + pickle_json(l, -1, entry, id); + + persist.push_back(std::move(entry)); + + lua_pop(l, 1); + } + + lua_pop(l, 1); + + json["lua_persistent_json"] = std::move(persist); + + LUA_DEBUG_END(l, 0); +} + +void LuaSerializer::LoadPersistent(const Json &json) +{ + lua_State *l = Lua::manager->GetLuaState(); + LUA_DEBUG_START(l); + + const Json &persist = json["lua_persistent_json"]; + + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_REFTABLE); + int idx_reftable = lua_gettop(l); + + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_PERSISTENT); + + for (const auto &item : persist) { + std::string id = item["persistId"].get(); + + lua_pushinteger(l, item["ref"].get()); + lua_getfield(l, -2, id.c_str()); // reftable, persist, ptr, pvalue + + // We must unpickle the saved persistent object, because it may contain + // "trapped" serialized representations of non-persistent tables + unpickle_json(l, item); + + // No valid persistent value (mismatch in serialization!) + if (lua_isnil(l, -2)) { + Log::Warning("Restoring missing persistent object '{}' from savefile", id); + lua_replace(l, -2); // reftable, persistent, ptr, svalue + } else { + lua_pop(l, 1); // reftable, persistent, ptr, pvalue + } + + // All references to the prior saved value are replaced with the persistent object + lua_settable(l, idx_reftable); + } + + lua_pop(l, 2); + + LUA_DEBUG_END(l, 0); +} + void LuaSerializer::ToJson(Json &jsonObj) { PROFILE_SCOPED() @@ -375,13 +456,7 @@ void LuaSerializer::ToJson(Json &jsonObj) lua_newtable(l); int savetable = lua_gettop(l); - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - if (lua_isnil(l, -1)) { - lua_pop(l, 1); - lua_newtable(l); - lua_pushvalue(l, -1); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - } + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_CALLBACKS); lua_pushnil(l); while (lua_next(l, -2) != 0) { @@ -398,7 +473,7 @@ void LuaSerializer::ToJson(Json &jsonObj) Json pickled; pickle_json(l, savetable, pickled); - jsonObj["lua_modules_json"] = pickled; + jsonObj["lua_modules_json"] = std::move(pickled); lua_pop(l, 1); @@ -428,13 +503,7 @@ void LuaSerializer::FromJson(const Json &jsonObj) if (!lua_istable(l, -1)) throw SavedGameCorruptException(); int savetable = lua_gettop(l); - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - if (lua_isnil(l, -1)) { - lua_pop(l, 1); - lua_newtable(l); - lua_pushvalue(l, -1); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - } + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_CALLBACKS); lua_pushnil(l); while (lua_next(l, -2) != 0) { @@ -525,7 +594,7 @@ void LuaSerializer::LoadComponents(const Json &jsonObj, Space *space) * * Example: * - * > Serializer.Register("MyModule", function() return {} end, function(data) ... end) + * > Serializer:Register("MyModule", function() return {} end, function(data) ... end) * * Parameters: * @@ -544,13 +613,7 @@ int LuaSerializer::l_register(lua_State *l) luaL_checktype(l, 3, LUA_TFUNCTION); // any type of function luaL_checktype(l, 4, LUA_TFUNCTION); // any type of function - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - if (lua_isnil(l, -1)) { - lua_pop(l, 1); - lua_newtable(l); - lua_pushvalue(l, -1); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerCallbacks"); - } + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_CALLBACKS); lua_newtable(l); @@ -605,13 +668,7 @@ int LuaSerializer::l_register_class(lua_State *l) return luaL_error(l, "Serializer class '%s' has no 'Unserialize' method", key.c_str()); lua_pop(l, 2); - lua_getfield(l, LUA_REGISTRYINDEX, "PiSerializerClasses"); - if (lua_isnil(l, -1)) { - lua_pop(l, 1); - lua_newtable(l); - lua_pushvalue(l, -1); - lua_setfield(l, LUA_REGISTRYINDEX, "PiSerializerClasses"); - } + luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_CLASSES); lua_pushvalue(l, 3); lua_setfield(l, -2, key.c_str()); @@ -623,6 +680,33 @@ int LuaSerializer::l_register_class(lua_State *l) return 0; } +/* + * Function: RegisterPersistent + * + * Register the given table as a "persistent value" which should not + * be serialized but instead have references to it in a saved game replaced + * with the value stored under the same ID at load-time. + * + * The passed value must be able to be serialized into a saved game, as the + * value will be loaded from the savefile to maintain backwards compatibility + * if no persistent object is registered with that ID at load time. + * + * Parameters: + * key - string, unique ID of the table to serialize + * value - table, persistent value to store + */ +int LuaSerializer::l_register_persistent(lua_State *l) +{ + luaL_checktype(l, 2, LUA_TSTRING); + luaL_checktype(l, 3, LUA_TTABLE); + + lua_getfield(l, LUA_REGISTRYINDEX, NS_PERSISTENT); + lua_replace(l, 1); + lua_settable(l, -3); + + return 0; +} + template <> const char *LuaObject::s_type = "Serializer"; @@ -636,9 +720,16 @@ void LuaObject::RegisterClass() static const luaL_Reg l_methods[] = { { "Register", LuaSerializer::l_register }, { "RegisterClass", LuaSerializer::l_register_class }, + { "RegisterPersistent", LuaSerializer::l_register_persistent }, { 0, 0 } }; + lua_newtable(l); + lua_setfield(l, LUA_REGISTRYINDEX, NS_PERSISTENT); + + lua_newtable(l); + lua_setfield(l, LUA_REGISTRYINDEX, NS_CLASSES); + lua_getfield(l, LUA_REGISTRYINDEX, "CoreImports"); LuaObjectBase::CreateObject(l_methods, 0, 0); lua_setfield(l, -2, "Serializer"); diff --git a/src/lua/LuaSerializer.h b/src/lua/LuaSerializer.h index 4b6fa3cf4c3..219bcd1040b 100644 --- a/src/lua/LuaSerializer.h +++ b/src/lua/LuaSerializer.h @@ -24,12 +24,16 @@ class LuaSerializer : public DeleteEmitter { void SaveComponents(Json &jsonObj, Space *space); void LoadComponents(const Json &jsonObj, Space *space); + void SavePersistent(Json &jsonObj); + void LoadPersistent(const Json &jsonObj); + void InitTableRefs(); void UninitTableRefs(); private: static int l_register(lua_State *l); static int l_register_class(lua_State *l); + static int l_register_persistent(lua_State *l); static void pickle_json(lua_State *l, int idx, Json &out, const std::string &key = ""); static void unpickle_json(lua_State *l, const Json &value); From ea17e3fbfcade66bc93e0baf8dc6e6745aa8709c Mon Sep 17 00:00:00 2001 From: Webster Sheets Date: Fri, 19 Jan 2024 14:01:21 -0500 Subject: [PATCH 2/4] Add persistent serialization testcase --- data/modules/Debug/TestSerialization.lua | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/data/modules/Debug/TestSerialization.lua b/data/modules/Debug/TestSerialization.lua index d73615c7541..03310fc58a7 100644 --- a/data/modules/Debug/TestSerialization.lua +++ b/data/modules/Debug/TestSerialization.lua @@ -44,3 +44,33 @@ end Serializer:RegisterClass('TestClass', testClass) Serializer:Register('TestSerialization', serialize, unserialize) --]] + +--[[ +local persistentClass = utils.inherits(nil, 'TestClassPersistent') + +local persistentInstance = persistentClass.New() + +function persistentClass:Unserialize() + print("Unserialized a persistent object") + + return setmetatable(self, persistentClass.meta) +end + +Serializer:RegisterPersistent('TestPersistentObject', persistentInstance) + +local function serialize() + local test_data = { + instance = persistentInstance + } + return test_data +end + +local function unserialize(data) + print(persistentInstance) + print(data.instance) + print(persistentInstance == data.instance) +end + +Serializer:RegisterClass('TestClassPersistent', persistentClass) +Serializer:Register('TestSerialization2', serialize, unserialize) +--]] From 834aeb55cc829f09662a0dc029e590ed2bad77b6 Mon Sep 17 00:00:00 2001 From: Webster Sheets Date: Fri, 23 Feb 2024 00:09:38 -0500 Subject: [PATCH 3/4] CommodityTypes are serialized as persistent objects --- data/libs/CommodityType.lua | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/data/libs/CommodityType.lua b/data/libs/CommodityType.lua index 9a82d6a648c..424a7155ca7 100644 --- a/data/libs/CommodityType.lua +++ b/data/libs/CommodityType.lua @@ -100,8 +100,12 @@ CommodityType.registry = {} function CommodityType.RegisterCommodity(name, info) assert(not CommodityType.registry[name]) - CommodityType.registry[name] = CommodityType.New(name, info) - return CommodityType.registry[name] + local commodity = CommodityType.New(name, info) + + CommodityType.registry[name] = commodity + Serializer:RegisterPersistent("CommodityType." .. name, commodity) + + return commodity end -- Function: GetCommodity @@ -122,14 +126,14 @@ end -- Ensure loaded commodity types always point at the 'canonical' instance of the commodity; -- commodity types not defined by the current version of the code will be loaded verbatim function CommodityType.Unserialize(data) - local ct = CommodityType.GetCommodity(data.name) + setmetatable(data, CommodityType.meta) - if not ct then + if not CommodityType.registry[data.name] then logWarning('Commodity type ' .. data.name .. ' could not be found, are you loading an outdated save?') - ct = CommodityType.RegisterCommodity(data.name, data) + CommodityType.registry[data.name] = data end - return ct + return data end Serializer:RegisterClass('CommodityType', CommodityType) From f9dd8c2a51b63c72ab0dd2347e69c4a6725405f4 Mon Sep 17 00:00:00 2001 From: Webster Sheets Date: Fri, 1 Mar 2024 17:48:17 -0500 Subject: [PATCH 4/4] Address review feedback - More clearly document the serialization process of persistent objects - Write missing persistent objects back to the persistent object cache so they are correctly serialized if the game is re-saved to disk --- src/lua/LuaSerializer.cpp | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/lua/LuaSerializer.cpp b/src/lua/LuaSerializer.cpp index 6da666fbd6e..b4752722cbf 100644 --- a/src/lua/LuaSerializer.cpp +++ b/src/lua/LuaSerializer.cpp @@ -391,7 +391,8 @@ void LuaSerializer::SavePersistent(Json &json) std::string id = LuaPull(l, -2); // Serialize all registered persistent objects as a "backup" version - // (This will populate the PiSerializerTableRefs cache) + // This will populate the table identity cache to avoid persistent + // objects being serialized twice. entry["persistId"] = id; pickle_json(l, -1, entry, id); @@ -418,21 +419,36 @@ void LuaSerializer::LoadPersistent(const Json &json) int idx_reftable = lua_gettop(l); luaL_getsubtable(l, LUA_REGISTRYINDEX, NS_PERSISTENT); + int idx_persist = lua_gettop(l); for (const auto &item : persist) { std::string id = item["persistId"].get(); lua_pushinteger(l, item["ref"].get()); - lua_getfield(l, -2, id.c_str()); // reftable, persist, ptr, pvalue - - // We must unpickle the saved persistent object, because it may contain - // "trapped" serialized representations of non-persistent tables + lua_getfield(l, idx_persist, id.c_str()); // reftable, persist, ptr, pvalue + + // Serialization order is first-in first-out - because persistent objects + // are serialized before all other objects, their serialized representation + // may contain the definition of "common subtables" referred to by other + // objects not serialized in the persistent table. + // Thus, (as a rule) we must always unserialize every object that was + // initially serialized, in serialization order, to populate the reference + // table and avoid any potential dangling references to serialized tables. unpickle_json(l, item); // No valid persistent value (mismatch in serialization!) if (lua_isnil(l, -2)) { Log::Warning("Restoring missing persistent object '{}' from savefile", id); lua_replace(l, -2); // reftable, persistent, ptr, svalue + + // Write this back to the persistent table so it will be included + // if the file is re-saved. + // TODO: because the Lua state is shared across save/load cycles, + // this value will "leak" into newly-started games. This is not + // something that can be feasibly addressed except by creating new + // lua_States on starting a new game. + lua_pushvalue(l, -1); // reftable, persistent, ptr, svalue, svalue + lua_setfield(l, idx_persist, id.c_str()); } else { lua_pop(l, 1); // reftable, persistent, ptr, pvalue }