Merge branch 'main' into self_hosted_runner

.env.example

@@ -42,7 +42,7 @@ SHARP_PROOF_LAYOUT=
 
 DA_LAYER=
 SETTLEMENT_LAYER=
-ETHEREUM_RPC_URL=
+SETTLEMENT_RPC_URL=
 MADARA_RPC_URL=
 MEMORY_PAGES_CONTRACT_ADDRESS=
 PRIVATE_KEY=

.env.test

@@ -25,6 +25,7 @@ SQS_JOB_HANDLE_FAILURE_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.clou
 SQS_WORKER_TRIGGER_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_worker_trigger_queue"
 
 ##### SNS #####
+
 ALERTS="sns"
 AWS_SNS_REGION="us-east-1"
 AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
@@ -50,8 +51,8 @@ SHARP_PROOF_LAYOUT="small"
 
 DA_LAYER="ethereum"
 SETTLEMENT_LAYER="ethereum"
-SETTLEMENT_RPC_URL="http://localhost:3001"
-MADARA_RPC_URL="http://localhost:3000"
+SETTLEMENT_RPC_URL="https://eth-mainnet.public.blastapi.io"
+MADARA_RPC_URL="http://localhost:5000"
 L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
 MEMORY_PAGES_CONTRACT_ADDRESS="0x47312450B3Ac8b5b8e247a6bB6d523e7605bDb60"
 PRIVATE_KEY="0xdead"
@@ -60,8 +61,4 @@ STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C
 DEFAULT_L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
 TEST_DUMMY_CONTRACT_ADDRESS="0xE5b6F5e695BA6E4aeD92B68c4CC8Df1160D69A81"
 STARKNET_OPERATOR_ADDRESS="0x2C169DFe5fBbA12957Bdd0Ba47d9CEDbFE260CA7"
-ETHEREUM_BLAST_RPC_URL="https://eth-mainnet.public.blastapi.io"
-
-##### E2E test vars #####
-
-L2_BLOCK_NUMBER_FOR_TEST=671070
+ETHEREUM_BLAST_RPC_URL="https://eth-mainnet.public.blastapi.io"

.github/workflows/coverage.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Run llvm-cov tests
         env:
           ETHEREUM_BLAST_RPC_URL: ${{ secrets.ETHEREUM_BLAST_RPC_URL }}
-        run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1
+        run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --features testing --lcov --output-path lcov.info --test-threads=1 --workspace --exclude=e2e-tests
 
       - name: Coveralls
         uses: coverallsapp/github-action@v2

.github/workflows/e2e-test.yml

@@ -0,0 +1,59 @@
+name: E2E test
+
+on:
+  workflow_call:
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+
+    services:
+      localstack:
+        image: localstack/localstack
+        env:
+          DEFAULT_REGION: us-east-1
+          AWS_ACCESS_KEY_ID: "AWS_ACCESS_KEY_ID"
+          AWS_SECRET_ACCESS_KEY: "AWS_SECRET_ACCESS_KEY"
+        ports:
+          - 4566:4566
+      mongodb:
+        image: mongo:latest
+        ports:
+          - 27017:27017
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # selecting a toolchain either by action or manual `rustup` calls should happen
+      # before the plugin, as the cache uses the current rustc version as its cache key
+      - run: rustup show
+
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Check Anvil Installation
+        run: |
+          if command -v anvil &> /dev/null
+          then
+              echo "Anvil is installed. Version information:"
+              anvil --version
+          else
+              echo "Anvil is not installed or not in PATH"
+              exit 1
+          fi
+
+      - name: Run e2e test
+        env:
+          ETHEREUM_BLAST_RPC_URL: ${{ secrets.ETHEREUM_BLAST_RPC_URL }}
+        run: |
+          RUST_LOG=trace cargo test --features testing test_orchestrator_workflow -- --nocapture

.github/workflows/linters-cargo.yml

@@ -21,3 +21,5 @@ jobs:
           cargo fmt -- --check
           cargo clippy --no-deps -- -D warnings
           cargo clippy --tests --no-deps -- -D warnings
+          cargo clippy --features testing --no-deps -- -D warnings
+          cargo clippy --features testing --tests --no-deps -- -D warnings

.github/workflows/pull-request.yml

@@ -24,5 +24,9 @@ jobs:
   coverage:
     name: Run coverage
     uses: ./.github/workflows/coverage.yml
-    needs: rust_build
+    secrets: inherit
+
+  e2e-test:
+    name: E2E test
+    uses: ./.github/workflows/e2e-test.yml
     secrets: inherit

CHANGELOG.md

@@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## Added
 
 - moved rust build and tests to self-hosted runner
+- e2e flow test
+- database timestamps
 - alerts module.
 - Tests for Settlement client.
 - Worker queues to listen for trigger events.
@@ -32,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Changed
 
+- settings provider
 - refactor AWS config usage and clean .env files
 - GitHub's coverage CI yml file for localstack and db testing.
 - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.
@@ -46,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Fixed
 
+- Calculate root hash logic and added a simple test for it.
 - Cargo.toml dependency reorg.
 - Get Fact Info logic.
 - Fixed state update worker logic as per the new implementation.

Cargo.toml

@@ -24,15 +24,31 @@ authors = ["Apoorv Sadana <@apoorvsadana>"]
 [workspace.dependencies]
 num = { version = "0.4.1" }
 async-trait = { version = "0.1.77" }
-alloy = { version = "0.2.1", features = ["full"] }
+alloy = { version = "0.2.1", features = [
+  "full",
+  "node-bindings",
+  "rpc-types-debug",
+  "rpc-types-trace",
+  "json-rpc",
+  "rpc-client",
+] }
+aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
+aws-sdk-s3 = { version = "1.38.0", features = ["behavior-version-latest"] }
+aws-sdk-eventbridge = { version = "1.41.0", features = [
+  "behavior-version-latest",
+] }
+aws-sdk-sqs = "1.36.0"
 axum = { version = "0.7.4" }
 axum-macros = "0.4.1"
 bincode = "1.3.3"
+bytes = "1.6.0"
 color-eyre = "0.6.2"
+chrono = "0.4.0"
 c-kzg = "1.0.0"
 dotenvy = "0.15.7"
 futures = "0.3.30"
-mongodb = { version = "2.8.1" }
+log = "0.4.21"
+mongodb = { version = "2.8.1", features = ["bson-chrono-0_4"] }
 omniqueue = { version = "0.2.0" }
 reqwest = { version = "0.12.7", features = [
   "rustls-tls",