Hot archive tx apply (#4603)

# Description

Resolves #4584

This PR adds support for the Hot Archive to TX apply. Specifically,
`InvokeHostFunctionOp` will check both the live BucketList and Hot
Archive BucketList for archived entries. The `RestoreFootprintOp` can
now restore entries from both the live BucketList and Hot Archive.

Additionally, restore meta has also changed for p23. The
`LEDGER_ENTRY_RESTORED` type has been added for the restore op. When an
entry is restored. both the `LedgerEntry` of the restored data and the
corresponding `TTL` entry are emitted as `LEDGER_ENTRY_RESTORED
LedgerEntryChangeType`. If the entry has not yet been evicted (such that
the entry and it's TTL still exist in the live BucketList), the
preexisting TTL value will be emitted as `LEDGER_ENTRY_STATE`. If the
entry has been evicted such that the TTL value has been previously
deleted, only a single change type of `LEDGER_ENTRY_RESTORED` will be
emitted for the new value. For the data being restored,
`LEDGER_ENTRY_STATE` is never emitted.

Rebased on #4585.

# Checklist
- [x] Reviewed the
[contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes)
document
- [x] Rebased on top of master (no merge commits)
- [x] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio
extension)
- [x] Compiles
- [x] Ran all tests
- [ ] If change impacts performance, include supporting evidence per the
[performance
document](https://github.com/stellar/stellar-core/blob/master/performance-eval/performance-eval.md)

src/bucket/BucketManager.cpp

@@ -9,8 +9,9 @@
 #include "bucket/BucketSnapshotManager.h"
 #include "bucket/BucketUtils.h"
 #include "bucket/HotArchiveBucket.h"
+#include "bucket/HotArchiveBucketList.h"
 #include "bucket/LiveBucket.h"
-#include "bucket/SearchableBucketList.h"
+#include "bucket/LiveBucketList.h"
 #include "crypto/BLAKE2.h"
 #include "crypto/Hex.h"
 #include "history/HistoryManager.h"

src/bucket/BucketManager.h

@@ -1,12 +1,11 @@
 #pragma once
 
 #include "bucket/BucketMergeMap.h"
-#include "bucket/HotArchiveBucketList.h"
-#include "bucket/LiveBucketList.h"
 #include "main/Config.h"
 #include "util/types.h"
 #include "xdr/Stellar-ledger.h"
 
+#include <filesystem>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -32,6 +31,9 @@ class AbstractLedgerTxn;
 class Application;
 class Bucket;
 class LiveBucketList;
+class HotArchiveBucketList;
+class BucketBase;
+class BucketIndex;
 class BucketSnapshotManager;
 class SearchableLiveBucketListSnapshot;
 struct BucketEntryCounters;
@@ -395,7 +397,6 @@ class BucketManager : NonMovableOrCopyable
     scheduleVerifyReferencedBucketsWork(HistoryArchiveState const& has);
 
     Config const& getConfig() const;
-
     void reportBucketEntryCountMetrics();
 };
 

src/bucket/FutureBucket.cpp

@@ -7,6 +7,7 @@
 // else.
 #include "util/asio.h" // IWYU pragma: keep
 
+#include "bucket/BucketListBase.h"
 #include "bucket/BucketManager.h"
 #include "bucket/FutureBucket.h"
 #include "bucket/HotArchiveBucket.h"

src/bucket/SearchableBucketList.cpp

@@ -5,6 +5,7 @@
 #include "bucket/SearchableBucketList.h"
 #include "bucket/BucketInputIterator.h"
 #include "bucket/BucketListSnapshotBase.h"
+#include "bucket/LiveBucketList.h"
 #include "util/GlobalChecks.h"
 
 #include <medida/timer.h>

src/bucket/test/BucketTestUtils.cpp

@@ -235,15 +235,27 @@ LedgerManagerForBucketTests::transferLedgerEntriesToBucketList(
                         ltxEvictions, lh.ledgerSeq, keys, initialLedgerVers,
                         mApp.getLedgerManager()
                             .getSorobanNetworkConfigForApply());
-
                 if (protocolVersionStartsFrom(
                         initialLedgerVers,
                         BucketBase::
                             FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
                 {
+                    std::vector<LedgerKey> restoredKeys;
+                    auto restoredKeysMap = ltx.getRestoredHotArchiveKeys();
+                    for (auto const& key : restoredKeysMap)
+                    {
+                        // Hot Archive does not track TTLs
+                        if (key.type() == CONTRACT_DATA ||
+                            key.type() == CONTRACT_CODE)
+                        {
+                            restoredKeys.emplace_back(key);
+                        }
+                    }
                     mApp.getBucketManager().addHotArchiveBatch(
-                        mApp, lh, evictedState.archivedEntries, {}, {});
+                        mApp, lh, evictedState.archivedEntries, restoredKeys,
+                        {});
                 }
+
                 if (ledgerCloseMeta)
                 {
                     ledgerCloseMeta->populateEvictedEntries(evictedState);

src/catchup/IndexBucketsWork.cpp

@@ -5,6 +5,7 @@
 #include "IndexBucketsWork.h"
 #include "bucket/BucketIndex.h"
 #include "bucket/BucketManager.h"
+#include "bucket/LiveBucket.h"
 #include "util/Fs.h"
 #include "util/Logging.h"
 #include "util/UnorderedSet.h"

src/ledger/LedgerManagerImpl.cpp

@@ -4,6 +4,7 @@
 
 #include "ledger/LedgerManagerImpl.h"
 #include "bucket/BucketManager.h"
+#include "bucket/HotArchiveBucketList.h"
 #include "bucket/LiveBucketList.h"
 #include "catchup/AssumeStateWork.h"
 #include "crypto/Hex.h"
@@ -44,6 +45,7 @@
 
 #include <fmt/format.h>
 
+#include "xdr/Stellar-ledger-entries.h"
 #include "xdr/Stellar-ledger.h"
 #include "xdr/Stellar-transaction.h"
 #include "xdrpp/types.h"
@@ -1790,8 +1792,19 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList(
                     initialLedgerVers,
                     BucketBase::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
             {
+                std::vector<LedgerKey> restoredKeys;
+                auto const& restoredKeyMap = ltx.getRestoredHotArchiveKeys();
+                for (auto const& key : restoredKeyMap)
+                {
+                    // TTL keys are not recorded in the hot archive BucketList
+                    if (key.type() == CONTRACT_DATA ||
+                        key.type() == CONTRACT_CODE)
+                    {
+                        restoredKeys.push_back(key);
+                    }
+                }
                 mApp.getBucketManager().addHotArchiveBatch(
-                    mApp, lh, evictedState.archivedEntries, {}, {});
+                    mApp, lh, evictedState.archivedEntries, restoredKeys, {});
             }
 
             if (ledgerCloseMeta)

src/ledger/LedgerTxn.cpp

@@ -16,12 +16,14 @@
 #include "main/Application.h"
 #include "transactions/TransactionUtils.h"
 #include "util/GlobalChecks.h"
+#include "util/UnorderedSet.h"
 #include "util/types.h"
 #include "xdr/Stellar-ledger-entries.h"
 #include <Tracy.hpp>
 #include <soci.h>
 
 #include <algorithm>
+#include <stdexcept>
 
 namespace stellar
 {
@@ -501,14 +503,16 @@ LedgerTxn::Impl::commit() noexcept
     maybeUpdateLastModifiedThenInvokeThenSeal([&](EntryMap const& entries) {
         // getEntryIterator has the strong exception safety guarantee
         // commitChild has the strong exception safety guarantee
-        mParent.commitChild(getEntryIterator(entries), mConsistency);
+        mParent.commitChild(getEntryIterator(entries), mRestoredKeys,
+                            mConsistency);
     });
 }
 
 void
-LedgerTxn::commitChild(EntryIterator iter, LedgerTxnConsistency cons) noexcept
+LedgerTxn::commitChild(EntryIterator iter, RestoredKeys const& restoredKeys,
+                       LedgerTxnConsistency cons) noexcept
 {
-    getImpl()->commitChild(std::move(iter), cons);
+    getImpl()->commitChild(std::move(iter), restoredKeys, cons);
 }
 
 static LedgerTxnConsistency
@@ -527,6 +531,7 @@ joinConsistencyLevels(LedgerTxnConsistency c1, LedgerTxnConsistency c2)
 
 void
 LedgerTxn::Impl::commitChild(EntryIterator iter,
+                             RestoredKeys const& restoredKeys,
                              LedgerTxnConsistency cons) noexcept
 {
     // Assignment of xdrpp objects does not have the strong exception safety
@@ -632,6 +637,24 @@ LedgerTxn::Impl::commitChild(EntryIterator iter,
         printErrorAndAbort("unknown fatal error during commit to LedgerTxn");
     }
 
+    for (auto const& key : restoredKeys.hotArchive)
+    {
+        auto [_, inserted] = mRestoredKeys.hotArchive.emplace(key);
+        if (!inserted)
+        {
+            printErrorAndAbort("restored hot archive entry already exists");
+        }
+    }
+
+    for (auto const& key : restoredKeys.liveBucketList)
+    {
+        auto [_, inserted] = mRestoredKeys.liveBucketList.emplace(key);
+        if (!inserted)
+        {
+            printErrorAndAbort("restored live BucketList entry already exists");
+        }
+    }
+
     // std::unique_ptr<...>::swap does not throw
     mHeader.swap(childHeader);
     mChild = nullptr;
@@ -802,6 +825,91 @@ LedgerTxn::Impl::erase(InternalLedgerKey const& key)
     }
 }
 
+void
+LedgerTxn::restoreFromHotArchive(LedgerEntry const& entry, uint32_t ttl)
+{
+    getImpl()->restoreFromHotArchive(*this, entry, ttl);
+}
+
+void
+LedgerTxn::Impl::restoreFromHotArchive(LedgerTxn& self,
+                                       LedgerEntry const& entry, uint32_t ttl)
+{
+    throwIfSealed();
+    throwIfChild();
+
+    if (!isPersistentEntry(entry.data))
+    {
+        throw std::runtime_error("Key type not supported in Hot Archive");
+    }
+    auto ttlKey = getTTLKey(entry);
+
+    // Restore entry by creating it on the live BucketList
+    create(self, entry);
+
+    // Also create the corresponding TTL entry
+    LedgerEntry ttlEntry;
+    ttlEntry.data.type(TTL);
+    ttlEntry.data.ttl().liveUntilLedgerSeq = ttl;
+    ttlEntry.data.ttl().keyHash = ttlKey.ttl().keyHash;
+    create(self, ttlEntry);
+
+    // Mark the keys as restored
+    auto addKey = [this](LedgerKey const& key) {
+        auto [_, inserted] = mRestoredKeys.hotArchive.insert(key);
+        if (!inserted)
+        {
+            throw std::runtime_error("Key already removed from hot archive");
+        }
+    };
+    addKey(LedgerEntryKey(entry));
+    addKey(ttlKey);
+}
+
+void
+LedgerTxn::restoreFromLiveBucketList(LedgerKey const& key, uint32_t ttl)
+{
+    getImpl()->restoreFromLiveBucketList(*this, key, ttl);
+}
+
+void
+LedgerTxn::Impl::restoreFromLiveBucketList(LedgerTxn& self,
+                                           LedgerKey const& key, uint32_t ttl)
+{
+    throwIfSealed();
+    throwIfChild();
+
+    if (!isPersistentEntry(key))
+    {
+        throw std::runtime_error("Key type not supported for restoration");
+    }
+
+    auto ttlKey = getTTLKey(key);
+
+    // Note: key should have already been loaded via loadWithoutRecord by
+    // caller, so this read should already be in the cache.
+    auto ttlLtxe = load(self, ttlKey);
+    if (!ttlLtxe)
+    {
+        throw std::runtime_error("Entry restored from live BucketList but does "
+                                 "not exist in the live BucketList.");
+    }
+
+    ttlLtxe.current().data.ttl().liveUntilLedgerSeq = ttl;
+
+    // Mark the keys as restored
+    auto addKey = [this](LedgerKey const& key) {
+        auto [_, inserted] = mRestoredKeys.liveBucketList.insert(key);
+        if (!inserted)
+        {
+            throw std::runtime_error(
+                "Key already restored from Live BucketList");
+        }
+    };
+    addKey(key);
+    addKey(ttlKey);
+}
+
 void
 LedgerTxn::eraseWithoutLoading(InternalLedgerKey const& key)
 {
@@ -1470,6 +1578,30 @@ LedgerTxn::Impl::getAllEntries(std::vector<LedgerEntry>& initEntries,
     deadEntries.swap(resDead);
 }
 
+UnorderedSet<LedgerKey> const&
+LedgerTxn::getRestoredHotArchiveKeys() const
+{
+    return getImpl()->getRestoredHotArchiveKeys();
+}
+
+UnorderedSet<LedgerKey> const&
+LedgerTxn::Impl::getRestoredHotArchiveKeys() const
+{
+    return mRestoredKeys.hotArchive;
+}
+
+UnorderedSet<LedgerKey> const&
+LedgerTxn::getRestoredLiveBucketListKeys() const
+{
+    return getImpl()->getRestoredLiveBucketListKeys();
+}
+
+UnorderedSet<LedgerKey> const&
+LedgerTxn::Impl::getRestoredLiveBucketListKeys() const
+{
+    return mRestoredKeys.liveBucketList;
+}
+
 LedgerKeySet
 LedgerTxn::getAllTTLKeysWithoutSealing() const
 {
@@ -1957,6 +2089,8 @@ LedgerTxn::Impl::rollback() noexcept
     }
 
     mEntry.clear();
+    mRestoredKeys.hotArchive.clear();
+    mRestoredKeys.liveBucketList.clear();
     mMultiOrderBook.clear();
     mActive.clear();
     mActiveHeader.reset();
@@ -2559,10 +2693,10 @@ LedgerTxnRoot::Impl::throwIfChild() const
 }
 
 void
-LedgerTxnRoot::commitChild(EntryIterator iter,
+LedgerTxnRoot::commitChild(EntryIterator iter, RestoredKeys const& restoredKeys,
                            LedgerTxnConsistency cons) noexcept
 {
-    mImpl->commitChild(std::move(iter), cons);
+    mImpl->commitChild(std::move(iter), restoredKeys, cons);
 }
 
 static void
@@ -2619,6 +2753,7 @@ LedgerTxnRoot::Impl::bulkApply(BulkLedgerEntryChangeAccumulator& bleca,
 
 void
 LedgerTxnRoot::Impl::commitChild(EntryIterator iter,
+                                 RestoredKeys const& restoredHotArchiveKeys,
                                  LedgerTxnConsistency cons) noexcept
 {
     ZoneScoped;