Merge pull request #155 from freespek/igor/aggregate-storage153

implement basic state aggregation

Author: konnov

solarkraft/src/aggregate.ts

@@ -0,0 +1,95 @@
+/**
+ * @license
+ * [Apache-2.0](https://github.com/freespek/solarkraft/blob/main/LICENSE)
+ */
+/**
+ * A command to aggregate the full contract state from the collected transactions.
+ * This command is potentially expensive, as the full contract state potentially
+ * grows much faster than individual transactions (think of multiple pair-to-pair
+ * token transfers between a large number of accounts). In the long run, it makes
+ * sense to collect the states from the history archives. For the time being,
+ * we aggregate the states from the transactions directly, in order to evaluate
+ * the approach.
+ *
+ * We need this feature primarily for input generation. This is an experimental
+ * feature in Phase 1. We always aggregate the state, starting with the minimal
+ * available height. Obviously, there is a lot of room for improvement here.
+ *
+ * Although it is tempting to aggregate transactions directly in fetch.ts,
+ * this is the wrong approach. Horizon may give us transactions out of order,
+ * so we need to sort them by height before state aggregation.
+ *
+ * Igor Konnov, 2024
+ */
+
+import { existsSync, writeFileSync } from 'fs'
+import { join } from 'path'
+import { JSONbig } from './globals.js'
+import {
+    emptyFullState,
+    loadContractCallEntry,
+    storagePath,
+    yieldListEntriesForContract,
+} from './fetcher/storage.js'
+import { applyCallToState } from './aggregator/aggregator.js'
+
+/**
+ * Aggregate the fetched transactions to compute the full contract state.
+ * @param args the aggregator arguments
+ */
+export async function aggregate(args: any) {
+    const storageRoot = storagePath(args.home)
+    if (!existsSync(storageRoot)) {
+        console.error(`The storage is empty. Run 'solarkraft fetch'`)
+        return
+    }
+
+    const contractId = args.id
+
+    // We have to sort the entries by height. Hence, we read them first and then sort.
+    let lastEntry = undefined
+    const entries = []
+    for (const e of yieldListEntriesForContract(
+        contractId,
+        join(storageRoot, contractId)
+    )) {
+        if (e.height <= args.heightTo) {
+            entries.push(e)
+        }
+        if (lastEntry && lastEntry.height === e.height) {
+            // this should not happen on the testnet, as there is only one transaction per height
+            console.warn(
+                `Height ${e.height}: transactions ${e.txHash} and ${lastEntry.txHash} may be out of order`
+            )
+        }
+        lastEntry = e
+    }
+    // sort the entries
+    entries.sort((a, b) => a.height - b.height)
+
+    // now we can aggregate the state
+    let nentries = 0
+    let state = emptyFullState()
+    for (const entry of entries) {
+        nentries++
+        const txEntry = loadContractCallEntry(args.home, entry.txHash)
+        if (txEntry.isRight()) {
+            if (args.verbose) {
+                console.log(`Height ${entry.height}: applied ${entry.txHash}`)
+            }
+            state = applyCallToState(state, txEntry.value)
+        } else {
+            console.error(
+                `Failed to load the transaction ${entry.txHash}: ${txEntry.value}`
+            )
+            return
+        }
+    }
+
+    // save the aggregated state
+    const contents = JSONbig.stringify(state)
+    writeFileSync(args.out, contents)
+    if (args.verbose) {
+        console.log(`Aggregated ${nentries} transactions into ${args.out}`)
+    }
+}

solarkraft/src/aggregator/aggregator.ts

@@ -0,0 +1,96 @@
+/**
+ * @license
+ * [Apache-2.0](https://github.com/freespek/solarkraft/blob/main/LICENSE)
+ */
+// State aggregator. We do not expect this aggregator to be efficient.
+// It is a proof of concept that is needed to implement input generation.
+//
+// See: https://github.com/freespek/solarkraft/issues/153
+//
+// Igor Konnov, 2024
+
+import {
+    ContractCallEntry,
+    emptyContractStorage,
+    FieldsMap,
+    FullState,
+    MultiContractStorage,
+} from '../fetcher/storage.js'
+
+/**
+ * Apply the updates from a contract call to the state.
+ * @param state the state to update
+ * @param callEntry the call entry to apply
+ * @returns the updated state
+ */
+export function applyCallToState(
+    state: FullState,
+    callEntry: ContractCallEntry
+): FullState {
+    if (callEntry.txSuccess !== true) {
+        console.warn(`Skipping failed transaction ${callEntry.txHash}`)
+        return state
+    }
+
+    return {
+        contractId: callEntry.contractId,
+        timestamp: callEntry.timestamp,
+        height: callEntry.height,
+        latestTxHash: callEntry.txHash,
+        storage: updateContractStorage(
+            state.storage,
+            callEntry.oldStorage,
+            callEntry.storage
+        ),
+    }
+}
+
+function updateContractStorage(
+    fullStorage: MultiContractStorage,
+    oldStorage: MultiContractStorage,
+    storage: MultiContractStorage
+): MultiContractStorage {
+    let updatedStorage = fullStorage
+    for (const [contractId, contractStorage] of storage) {
+        const contractFullStorage =
+            fullStorage.get(contractId) ?? emptyContractStorage()
+        const contractOldStorage =
+            oldStorage.get(contractId) ?? emptyContractStorage()
+        updatedStorage = updatedStorage.set(contractId, {
+            instance: updateFieldsMap(
+                contractFullStorage.instance,
+                contractOldStorage.instance,
+                contractStorage.instance
+            ),
+            persistent: updateFieldsMap(
+                contractFullStorage.persistent,
+                contractOldStorage.persistent,
+                contractStorage.persistent
+            ),
+            temporary: updateFieldsMap(
+                contractFullStorage.temporary,
+                contractOldStorage.temporary,
+                contractStorage.temporary
+            ),
+        })
+    }
+    return updatedStorage
+}
+
+function updateFieldsMap(
+    fullStorageFields: FieldsMap,
+    oldFieldsInCall: FieldsMap,
+    fieldsInCall: FieldsMap
+): FieldsMap {
+    let updatedFields = fullStorageFields
+    // note that storage entries in ContractCallEntry are small subsets of the full storage
+    for (const key of oldFieldsInCall.keys()) {
+        if (!fieldsInCall.has(key)) {
+            updatedFields = updatedFields.delete(key)
+        }
+    }
+    for (const [key, value] of fieldsInCall) {
+        updatedFields = updatedFields.set(key, value)
+    }
+    return updatedFields
+}

solarkraft/src/fetcher/storage.ts

@@ -178,6 +178,56 @@ export interface FetcherState {
     heights: OrderedMap<string, number>
 }
 
+/**
+ * This entry collects the full contract state.
+ * This is obviously expensive to store, so it should be improved,
+ * if the MVP proves successful.
+ */
+export interface FullState {
+    /**
+     * The number of seconds elapsed since unix epoch of when the latest contract ledger was closed.
+     */
+    timestamp: number
+    /**
+     * The ledger number that this state corresponds to.
+     */
+    height: number
+    /**
+     * The hash of the latest transaction that led to this state.
+     */
+    latestTxHash: string
+    /**
+     * The address of the contract being called.
+     */
+    contractId: string
+    /**
+     * Ordered mapping from contract address to instance/persistent/temporary storage
+     * of the respective contract. The contract storage for a given durability is
+     * an ordered mapping from field names to their native values (JS).
+     *
+     * This mapping contains values only for the fields that have been created
+     * or updated by a transaction in the past. It may happen that
+     * `storage` contains fewer fields than `oldStorage`, when the contract
+     * deletes some fields from the storage. Also, fields may be cleared from `storage`
+     * when the storage goes over TTL.
+     */
+    storage: MultiContractStorage
+}
+
+/**
+ * The empty full state that is to be initialized.
+ * @returns an empty full state
+ */
+export function emptyFullState(): FullState {
+    return {
+        timestamp: 0,
+        height: 0,
+        latestTxHash: '',
+        contractId: '',
+        storage: emptyMultiContractStorage(),
+    }
+}
+
 /**
  * Given the solarkraft home, construct the path to store the transactions.
  * @param solarkraftHome path to solarkraft home (or project directory)
@@ -191,8 +241,9 @@ export function storagePath(solarkraftHome: string): string {
  * Store a contract call entry in the file storage.
  * @param home the storage root directory
  * @param entry a call entry
+ * @returns the filename, where the entry was stored
  */
-export function saveContractCallEntry(home: string, entry: ContractCallEntry) {
+export function saveContractCallEntry(home: string, entry: ContractCallEntry): string {
     const filename = getEntryFilename(storagePath(home), entry)
     const verificationStatus: VerificationStatus =
         entry.verificationStatus ?? VerificationStatus.Unknown
@@ -261,6 +312,44 @@ export function loadContractCallEntry(
     })
 }
 
+/**
+ * Load full state of a contract from the storage.
+ * @param solarkraftHome the .solarkraft directory
+ * @param contractId the contract address
+ * @returns the loaded full state
+ */
+export function loadContractFullState(
+    solarkraftHome: string,
+    contractId: string
+): Result<FullState> {
+    const filename = getFullStateFilename(
+        storagePath(solarkraftHome),
+        contractId
+    )
+    if (!existsSync(filename)) {
+        return left(`No state found for contract ${contractId}`)
+    }
+    const contents = readFileSync(filename)
+    const loaded = JSONbig.parse(contents)
+    return right({
+        ...loaded,
+        storage: storageFromJS(loaded.storage),
+    })
+}
+
+/**
+ * Store contract full state in the file storage.
+ * @param home the storage root directory
+ * @param state a state of the contract
+ * @returns the filename, where the state was stored
+ */
+export function saveContractFullState(home: string, state: FullState): string {
+    const filename = getFullStateFilename(storagePath(home), state.contractId)
+    const contents = JSONbig.stringify(state)
+    writeFileSync(filename, contents)
+    return filename
+}
+
 /**
  * Generate storage entries for a given contract id in a path.
  * @param contractId contract identifier (address)
@@ -275,7 +364,8 @@ export function* yieldListEntriesForContract(
         if (dirent.isDirectory() && /^[0-9]+$/.exec(dirent.name)) {
             // This directory may contain several transactions for the same height.
             const height = Number.parseInt(dirent.name)
-            for (const ledgerDirent of readdirSync(join(path, dirent.name), {
+            const dirPath = join(path, dirent.name)
+            for (const ledgerDirent of readdirSync(dirPath, {
                 withFileTypes: true,
             })) {
                 // match all storage entries, which may be reported in different cases
@@ -284,10 +374,7 @@ export function* yieldListEntriesForContract(
                 )
                 if (ledgerDirent.isFile() && matcher) {
                     const txHash = matcher[1]
-                    const filename = join(
-                        ledgerDirent.path,
-                        `entry-${txHash}.json`
-                    )
+                    const filename = join(dirPath, `entry-${txHash}.json`)
                     const contents = JSONbig.parse(
                         readFileSync(filename, 'utf-8')
                     )
@@ -351,10 +438,22 @@ export function saveFetcherState(home: string, state: FetcherState): string {
  * @returns the filename
  */
 function getEntryFilename(root: string, entry: ContractCallEntry) {
-    const dir = getOrCreateDirectory(root, entry)
+    const dir = getOrCreateEntryParentDir(root, entry)
     return join(dir, `entry-${entry.txHash}.json`)
 }
 
+/**
+ * Get the directory name to store contract state.
+ *
+ * @param root storage root
+ * @param contractId the contract id to retrieve the state for
+ * @returns the directory name
+ */
+function getFullStateFilename(root: string, contractId: string) {
+    const dir = getOrCreateContractDir(root, contractId)
+    return join(dir, 'state.json')
+}
+
 /**
  * Get the filename for the fetcher state.
  *
@@ -374,8 +473,23 @@ function getFetcherStateFilename(root: string) {
  * @param entry call entry
  * @returns the directory name
  */
-function getOrCreateDirectory(root: string, entry: ContractCallEntry) {
-    const directory = join(root, entry.contractId, entry.height.toString())
+function getOrCreateEntryParentDir(root: string, entry: ContractCallEntry) {
+    const contractDir = getOrCreateContractDir(root, entry.contractId)
+    const directory = join(contractDir, entry.height.toString())
+    mkdirSync(directory, { recursive: true })
+    return directory
+}
+
+/**
+ * Return the contract directory.
+ * If this directory does not exist, create it recursively.
+ *
+ * @param root storage root
+ * @param contractId contract address
+ * @returns the directory name
+ */
+function getOrCreateContractDir(root: string, contractId: string) {
+    const directory = join(root, contractId)
     mkdirSync(directory, { recursive: true })
     return directory
 }