feat(ethexe): handle freed pages of programs memory (#4302)

Makefile

@@ -6,6 +6,7 @@ ethexe-pre-commit: ethexe-contracts-pre-commit ethexe-pre-commit-no-contracts
 ethexe-pre-commit-no-contracts:
 	@ echo " > Formatting ethexe" && cargo +nightly fmt --all -- --config imports_granularity=Crate,edition=2021
 	@ echo " >> Clippy checking ethexe" && cargo clippy -p "ethexe-*" --all-targets --all-features -- --no-deps -D warnings
+	@ echo " >>> Testing ethexe" && cargo test -p "ethexe-*"
 
 # Building ethexe contracts
 .PHONY: ethexe-contracts-pre-commit

ethexe/cli/src/config.rs

@@ -19,8 +19,7 @@
 //! Application config in one place.
 
 use crate::args::Args;
-
-use anyhow::{Context as _, Result};
+use anyhow::{ensure, Context as _, Result};
 use directories::ProjectDirs;
 use ethexe_network::NetworkEventLoopConfig;
 use ethexe_prometheus_endpoint::Registry;
@@ -173,14 +172,14 @@ impl TryFrom<Args> for Config {
 
         let sequencer =
             ConfigPublicKey::new(&args.sequencer_key).context("invalid sequencer key")?;
-        anyhow::ensure!(
+        ensure!(
             args.tmp || sequencer != ConfigPublicKey::Random,
             "random key for sequencer is only allowed with `--tmp` flag"
         );
 
         let validator =
             ConfigPublicKey::new(&args.validator_key).context("invalid validator key")?;
-        anyhow::ensure!(
+        ensure!(
             args.tmp || validator != ConfigPublicKey::Random,
             "random key for validator is only allowed with `--tmp` flag"
         );

ethexe/processor/src/handling/events.rs

@@ -17,7 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::Processor;
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, ensure, Result};
 use ethexe_common::{
     mirror::RequestEvent as MirrorEvent,
     router::{RequestEvent as RouterEvent, ValueClaim},
@@ -272,12 +272,12 @@ impl Processor {
     }
 
     pub fn handle_new_program(&mut self, program_id: ProgramId, code_id: CodeId) -> Result<()> {
-        anyhow::ensure!(
+        ensure!(
             self.db.original_code(code_id).is_some(),
             "code existence must be checked on router"
         );
 
-        anyhow::ensure!(
+        ensure!(
             self.db.program_code_id(program_id).is_none(),
             "program duplicates must be checked on router"
         );

ethexe/processor/src/handling/mod.rs

@@ -17,7 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::Processor;
-use anyhow::Result;
+use anyhow::{ensure, Result};
 use ethexe_db::CodesStorage;
 use ethexe_runtime_common::{
     state::{ComplexStorage as _, Dispatch},
@@ -65,7 +65,7 @@ impl Processor {
         }
 
         self.db.mutate_state(state_hash, |processor, state| {
-            anyhow::ensure!(state.program.is_active(), "program should be active");
+            ensure!(state.program.is_active(), "program should be active");
 
             state.queue_hash = processor
                 .modify_queue(state.queue_hash.clone(), |queue| queue.extend(dispatches))?;

ethexe/processor/src/lib.rs

@@ -18,7 +18,7 @@
 
 //! Program's execution service for eGPU.
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, ensure, Result};
 use ethexe_common::{mirror::RequestEvent as MirrorEvent, BlockRequestEvent};
 use ethexe_db::{BlockMetaStorage, CodesStorage, Database};
 use ethexe_runtime_common::{state::Storage, InBlockTransitions};
@@ -176,7 +176,7 @@ impl OverlaidProcessor {
             .read_state(state_hash)
             .ok_or_else(|| anyhow!("unreachable: state partially presents in storage"))?;
 
-        anyhow::ensure!(
+        ensure!(
             !state.requires_init_message(),
             "program isn't yet initialized"
         );

ethexe/runtime/common/src/journal.rs

@@ -6,7 +6,7 @@ use crate::{
     InBlockTransitions,
 };
 use alloc::{collections::BTreeMap, vec, vec::Vec};
-use anyhow::Result;
+use anyhow::{bail, Result};
 use core::num::NonZero;
 use core_processor::{
     common::{DispatchOutcome, JournalHandler},
@@ -83,15 +83,13 @@ impl<S: Storage> JournalHandler for Handler<'_, S> {
                 self.update_state(program_id, |state| {
                     match &mut state.program {
                         Program::Active(ActiveProgram { initialized, .. }) if *initialized => {
-                            anyhow::bail!("an attempt to initialize already initialized program")
+                            bail!("an attempt to initialize already initialized program")
                         }
                         Program::Active(ActiveProgram {
                             ref mut initialized,
                             ..
                         }) => *initialized = true,
-                        _ => anyhow::bail!(
-                            "an attempt to dispatch init message for inactive program"
-                        ),
+                        _ => bail!("an attempt to dispatch init message for inactive program"),
                     };
 
                     Ok(())
@@ -360,7 +358,7 @@ impl<S: Storage> JournalHandler for Handler<'_, S> {
                 ref mut pages_hash, ..
             }) = state.program
             else {
-                anyhow::bail!("an attempt to update pages data of inactive program");
+                bail!("an attempt to update pages data of inactive program");
             };
 
             let new_pages = storage.store_pages(pages_data);
@@ -383,18 +381,32 @@ impl<S: Storage> JournalHandler for Handler<'_, S> {
         self.update_state_with_storage(program_id, |storage, state| {
             let Program::Active(ActiveProgram {
                 ref mut allocations_hash,
+                ref mut pages_hash,
                 ..
             }) = state.program
             else {
-                anyhow::bail!("an attempt to update allocations of inactive program");
+                bail!("an attempt to update allocations of inactive program");
             };
 
-            // TODO (breathx): remove data for difference pages.
+            let mut removed_pages = vec![];
+
             *allocations_hash =
                 storage.modify_allocations(allocations_hash.clone(), |allocations| {
+                    removed_pages = allocations
+                        .difference(&new_allocations)
+                        .flat_map(|i| i.iter())
+                        .flat_map(|i| i.to_iter())
+                        .collect();
+
                     *allocations = new_allocations;
                 })?;
 
+            *pages_hash = storage.modify_memory_pages(pages_hash.clone(), |pages| {
+                for page in removed_pages {
+                    pages.remove(&page);
+                }
+            })?;
+
             Ok(())
         });
     }

ethexe/signer/src/lib.rs

@@ -25,7 +25,7 @@ pub use digest::{Digest, ToDigest};
 pub use sha3;
 pub use signature::Signature;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use gprimitives::{ActorId, H160};
 use parity_scale_codec::{Decode, Encode};
 use sha3::Digest as _;
@@ -205,7 +205,7 @@ impl Signer {
             }
         }
 
-        anyhow::bail!("Address not found: {}", address);
+        bail!("Address not found: {}", address);
     }
 
     pub fn get_key_by_addr(&self, address: Address) -> Result<Option<PublicKey>> {
@@ -279,7 +279,7 @@ impl Signer {
         let bytes = fs::read(key_path)?;
 
         if bytes.len() != 32 {
-            anyhow::bail!("Invalid key length: {:?}", bytes);
+            bail!("Invalid key length: {:?}", bytes);
         }
 
         buf.copy_from_slice(&bytes);

utils/calc-stack-height/src/main.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use anyhow::anyhow;
+use anyhow::{anyhow, ensure};
 use gear_core::code::{Code, TryNewCodeConfig};
 use gear_wasm_instrument::{SystemBreakCode, STACK_HEIGHT_EXPORT_NAME};
 use std::{env, fs};
@@ -142,7 +142,7 @@ fn main() -> anyhow::Result<()> {
     );
 
     if let Some(schedule_stack_height) = schedule.limits.stack_height {
-        anyhow::ensure!(
+        ensure!(
             schedule_stack_height <= stack_height,
             "Stack height in runtime schedule must be decreased"
         );

utils/wasm-builder/src/crate_info.rs

@@ -16,12 +16,11 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use anyhow::{Context, Result};
+use crate::{builder_error::BuilderError, multiple_crate_versions};
+use anyhow::{ensure, Context, Result};
 use cargo_metadata::{Metadata, MetadataCommand, Package};
 use std::{collections::BTreeMap, path::Path};
 
-use crate::{builder_error::BuilderError, multiple_crate_versions};
-
 /// Helper to get a crate info extracted from the `Cargo.toml`.
 #[derive(Debug, Default)]
 pub struct CrateInfo {
@@ -38,7 +37,7 @@ pub struct CrateInfo {
 impl CrateInfo {
     /// Create a new `CrateInfo` from a path to the `Cargo.toml`.
     pub fn from_manifest(manifest_path: &Path) -> Result<Self> {
-        anyhow::ensure!(
+        ensure!(
             manifest_path.exists(),
             BuilderError::ManifestPathInvalid(manifest_path.to_path_buf())
         );

utils/wasm-optimizer/src/cargo_toolchain.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, ensure, Context, Result};
 use regex::Regex;
 use std::{borrow::Cow, process::Command, sync::LazyLock};
 
@@ -48,7 +48,7 @@ impl Toolchain {
             .output()
             .context("`rustup` command failed")?;
 
-        anyhow::ensure!(
+        ensure!(
             output.status.success(),
             "`rustup` exit code is not successful"
         );
@@ -90,7 +90,7 @@ impl Toolchain {
     /// Checks whether the toolchain is recommended.
     pub fn check_recommended_toolchain(&self) -> Result<()> {
         let toolchain = Self::PINNED_NIGHTLY_TOOLCHAIN;
-        anyhow::ensure!(
+        ensure!(
             self.raw_toolchain_str() == toolchain,
             anyhow!(
                 "recommended toolchain `{x}` not found, install it using the command:\n\