From f9ef680f5b65eeff1987689fb228d798808ee9cb Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Mon, 16 Dec 2024 22:18:11 -0500 Subject: [PATCH] ci: add rust_ci.yaml github workflow (#15) * ci: add rust_ci.yaml github workflow * ci: remove benchmark from rust_ci since we don't have any * test: add dummy test in eigenda_data just to make `just test` not complain * style: fix lint issues * style: beautify justfile layout to make it easier to read * ci: add unused deps job to rust_ci workflow * ci: remove build job from rust_ci workflow * chore: fix justfile (lint was being redefined twice) * ci: fix unused-deps job, need to install machete, not unused-deps * ci: properly install cargo-machete in rust_ci * ci: wrong name for cargo-udeps * ci: increase unused-deps job timeout to 5 mins * style: stringify EigenDACommitment hint as "eigenda-commitment" instead of "altda-commitment" --- .github/workflows/rust_ci.yaml | 108 ++++++++++++++++++++++++++ Cargo.toml | 2 +- bin/host/src/eigenda_blobs.rs | 2 +- bin/host/src/fetcher/mod.rs | 2 +- bin/host/src/lib.rs | 2 - crates/eigenda/src/eigenda_blobs.rs | 64 +++++++-------- crates/eigenda/src/eigenda_data.rs | 33 ++++++++ crates/eigenda/src/traits.rs | 6 +- crates/proof/src/eigenda_provider.rs | 6 +- crates/proof/src/hint.rs | 8 +- justfile | 112 +++++++++++++++------------ 11 files changed, 250 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/rust_ci.yaml diff --git a/.github/workflows/rust_ci.yaml b/.github/workflows/rust_ci.yaml new file mode 100644 index 0000000..f9da74a --- /dev/null +++ b/.github/workflows/rust_ci.yaml @@ -0,0 +1,108 @@ +name: Rust CI + +on: + push: + branches: [master] + merge_group: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + cargo-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + name: test + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: taiki-e/install-action@just + - name: Install Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: taiki-e/install-action@nextest + - name: cargo test + run: just test + cargo-lint: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + matrix: + # TODO: add back cannon and asterisc once we are using them + # See https://github.com/Layr-Labs/kona/blob/main/.github/workflows/rust_ci.yaml + target: ["native"] + name: lint-${{ matrix.target }} + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: taiki-e/install-action@just + - name: Install Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + prefix-key: ${{ matrix.target }} + - name: Log into ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: fmt + lint + run: just lint-${{ matrix.target }} + - name: chown target + run: | + sudo chown -R $(id -u):$(id -g) ./target + cargo-doc: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: taiki-e/install-action@just + - name: Install Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: doclint + run: just lint-docs + - name: doctest + run: just test-docs + cargo-hack: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: taiki-e/install-action@just + - name: Install Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@cargo-hack + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: cargo hack + run: just hack + unused-deps: + runs-on: ubuntu-latest + timeout-minutes: 5 # machete runs very quickly but need longer timeout to install the tools + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: taiki-e/install-action@v2 + with: + tool: just, cargo-udeps + - name: Install cargo-machete + run: cargo install cargo-machete + - name: Install Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - run: just unused-deps diff --git a/Cargo.toml b/Cargo.toml index 1408458..130a138 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["bin/*"] +members = ["bin/*", "crates/*"] [workspace.dependencies] # Workspace diff --git a/bin/host/src/eigenda_blobs.rs b/bin/host/src/eigenda_blobs.rs index cbc0e51..f9c675b 100644 --- a/bin/host/src/eigenda_blobs.rs +++ b/bin/host/src/eigenda_blobs.rs @@ -2,7 +2,7 @@ use alloy_primitives::Bytes; use anyhow::Ok; use reqwest; -/// An online implementation of the [EigenDABlobProvider] trait. +/// Fetches blobs from EigenDA via an eigenda-proxy instance. #[derive(Debug, Clone)] pub struct OnlineEigenDABlobProvider { /// The base url. diff --git a/bin/host/src/fetcher/mod.rs b/bin/host/src/fetcher/mod.rs index 8993882..24a908d 100644 --- a/bin/host/src/fetcher/mod.rs +++ b/bin/host/src/fetcher/mod.rs @@ -569,7 +569,7 @@ where kv_write_lock.set(key.into(), preimage.into())?; } } - HintType::AltDACommitment => { + HintType::EigenDACommitment => { let cert = hint_data; info!(target: "fetcher", "Fetching AltDACommitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. diff --git a/bin/host/src/lib.rs b/bin/host/src/lib.rs index 5881630..6c6a6b1 100644 --- a/bin/host/src/lib.rs +++ b/bin/host/src/lib.rs @@ -12,8 +12,6 @@ use kona_host::cli::HostCli; use kona_host::kv; -use hokulea_client; - use crate::eigenda_blobs::OnlineEigenDABlobProvider; use anyhow::{anyhow, Result}; use fetcher::Fetcher; diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index c8e023e..1fb3957 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -37,6 +37,38 @@ where } } + /// Fetches the next blob from the source. + pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { + info!(target: "eigenda-blobsource", "next"); + self.load_blobs(altda_commitment).await?; + info!(target: "eigenda-blobsource", "next 1"); + let next_data = match self.next_data() { + Ok(d) => d, + Err(e) => return e, + }; + info!(target: "eigenda-blobsource", "next 2"); + // Decode the blob data to raw bytes. + // Otherwise, ignore blob and recurse next. + match next_data.decode() { + Ok(d) => { + info!(target: "eigenda-blobsource", "next 3"); + Ok(d) + } + Err(_) => { + warn!(target: "blob-source", "Failed to decode blob data, skipping"); + panic!() + // todo need to add recursion + // self.next(altda_commitment).await + } + } + } + + /// Clears the source. + pub fn clear(&mut self) { + self.data.clear(); + self.open = false; + } + /// Loads blob data into the source if it is not open. async fn load_blobs(&mut self, altda_commitment: &Bytes) -> Result<(), BlobProviderError> { if self.open { @@ -60,7 +92,7 @@ where } Err(_) => { self.open = true; - return Ok(()); + Ok(()) } } } @@ -73,34 +105,4 @@ where } Ok(self.data.remove(0)) } - - pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { - info!(target: "eigenda-blobsource", "next"); - self.load_blobs(altda_commitment).await?; - info!(target: "eigenda-blobsource", "next 1"); - let next_data = match self.next_data() { - Ok(d) => d, - Err(e) => return e, - }; - info!(target: "eigenda-blobsource", "next 2"); - // Decode the blob data to raw bytes. - // Otherwise, ignore blob and recurse next. - match next_data.decode() { - Ok(d) => { - info!(target: "eigenda-blobsource", "next 3"); - Ok(d) - } - Err(_) => { - warn!(target: "blob-source", "Failed to decode blob data, skipping"); - panic!() - // todo need to add recursion - // self.next(altda_commitment).await - } - } - } - - pub fn clear(&mut self) { - self.data.clear(); - self.open = false; - } } diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 754f3a2..7a60e9d 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -3,6 +3,7 @@ use alloy_primitives::Bytes; use kona_derive::errors::BlobDecodingError; #[derive(Default, Clone, Debug)] +/// Represents the data structure for EigenDA Blob. pub struct EigenDABlobData { /// The calldata pub(crate) blob: Bytes, @@ -17,3 +18,35 @@ impl EigenDABlobData { Ok(self.blob.clone()) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloy_primitives::Bytes; + + #[test] + fn test_decode_success() { + let data = EigenDABlobData { + blob: Bytes::from(vec![1, 2, 3, 4]), + }; + let result = data.decode(); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(vec![1, 2, 3, 4])); + } + + #[test] + fn test_decode_empty_blob() { + let data = EigenDABlobData { + blob: Bytes::from(vec![]), + }; + let result = data.decode(); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(vec![])); + } + + #[test] + fn test_decode_invalid_blob() { + // TODO: implement this once decode actually does something + } +} diff --git a/crates/eigenda/src/traits.rs b/crates/eigenda/src/traits.rs index 2fbd23e..5fad440 100644 --- a/crates/eigenda/src/traits.rs +++ b/crates/eigenda/src/traits.rs @@ -4,12 +4,16 @@ use async_trait::async_trait; use core::fmt::Display; use kona_derive::errors::PipelineErrorKind; +/// A trait for providing EigenDA blobs. +/// TODO: add explanation for why we need this to be a trait. #[async_trait] pub trait EigenDABlobProvider { - /// The error type for the [EigenDAProvider]. + /// The error type for the [EigenDABlobProvider]. type Error: Display + ToString + Into; + /// Fetches a blob. async fn get_blob(&mut self, cert: &Bytes) -> Result; + /// Fetches an element from a blob. async fn get_element(&mut self, cert: &Bytes, element: &Bytes) -> Result; } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 14647bd..63a81f4 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -29,7 +29,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider async fn get_blob(&mut self, cert: &Bytes) -> Result { self.oracle - .write(&HintType::AltDACommitment.encode_with(&[&cert])) + .write(&HintType::EigenDACommitment.encode_with(&[cert])) .await .map_err(OracleProviderError::Preimage)?; let data = self @@ -45,14 +45,14 @@ impl EigenDABlobProvider for OracleEigenDAProvider async fn get_element(&mut self, cert: &Bytes, element: &Bytes) -> Result { self.oracle - .write(&HintType::AltDACommitment.encode_with(&[&cert])) + .write(&HintType::EigenDACommitment.encode_with(&[cert])) .await .map_err(OracleProviderError::Preimage)?; let cert_point_key = Bytes::copy_from_slice(&[cert.to_vec(), element.to_vec()].concat()); self.oracle - .write(&HintType::AltDACommitment.encode_with(&[&cert_point_key])) + .write(&HintType::EigenDACommitment.encode_with(&[&cert_point_key])) .await .map_err(OracleProviderError::Preimage)?; let data = self diff --git a/crates/proof/src/hint.rs b/crates/proof/src/hint.rs index 75a6581..c099534 100644 --- a/crates/proof/src/hint.rs +++ b/crates/proof/src/hint.rs @@ -79,8 +79,8 @@ pub enum HintType { /// A hint that specifies bulk storage of all the code, state and keys generated by an /// execution witness. L2PayloadWitness, - /// - AltDACommitment, + /// A hint that specifies the EigenDA commitment. + EigenDACommitment, } impl HintType { @@ -109,7 +109,7 @@ impl TryFrom<&str> for HintType { "l2-account-proof" => Ok(Self::L2AccountProof), "l2-account-storage-proof" => Ok(Self::L2AccountStorageProof), "l2-payload-witness" => Ok(Self::L2PayloadWitness), - "altda-commitment" => Ok(Self::AltDACommitment), + "eigenda-commitment" => Ok(Self::EigenDACommitment), _ => Err(HintParsingError(value.to_string())), } } @@ -131,7 +131,7 @@ impl From for &str { HintType::L2AccountProof => "l2-account-proof", HintType::L2AccountStorageProof => "l2-account-storage-proof", HintType::L2PayloadWitness => "l2-payload-witness", - HintType::AltDACommitment => "altda-commitment", + HintType::EigenDACommitment => "eigenda-commitment", } } } diff --git a/justfile b/justfile index 684b6ba..c245655 100644 --- a/justfile +++ b/justfile @@ -1,14 +1,11 @@ set positional-arguments -alias l := lint-native -alias lint := lint-native -alias f := fmt-native-fix -alias b := build -alias h := hack # default recipe to display help information default: @just --list +############################### STYLE ############################### + # unused-deps finds unused dependencies in the workspace. # See https://rustprojectprimer.com/checks/unused.html # machete runs very fast but is less accurate, on by default. @@ -17,6 +14,45 @@ unused-deps slow="false": cargo machete # cargo +nightly udeps +# Lint the workspace for all available targets +alias la := lint +lint: lint-native lint-docs + +# Lint the workspace +alias l := lint-native +lint-native: fmt-native-check lint-docs + cargo +nightly clippy --workspace --all --all-features --all-targets -- -D warnings + +# Lint the Rust documentation +lint-docs: + RUSTDOCFLAGS="-D warnings" cargo doc --all --no-deps --document-private-items + +# Runs `cargo hack check` against the workspace +alias h := hack +hack: + cargo hack check --feature-powerset --no-dev-deps + +# Fixes the formatting of the workspace +alias f := fmt-native-fix +fmt-native-fix: + cargo +nightly fmt --all + +# Check the formatting of the workspace +fmt-native-check: + cargo +nightly fmt --all -- --check + +############################### BUILD ############################### + +# Build the workspace for all available targets +alias b := build +build: build-native + +# Build for the native target +build-native *args='': + cargo build --workspace $@ + +############################### UNIT TESTS ############################### + # Run all tests (excluding online tests) alias t := tests tests: test test-docs @@ -26,9 +62,28 @@ test *args="-E '!test(test_online)'": cargo nextest run --workspace --all --all-features {{args}} # Run all online tests +# TODO: understand when this is needed test-online: just test "-E 'test(test_online)'" +# Test the Rust documentation +test-docs: + cargo test --doc --all --locked + +############################### E2E TESTS ############################### + +# TODO: Use the below stuff to add an e2e test for the client program using hokulea + +# Clones and checks out the monorepo at the commit present in `.monorepo` +monorepo: + ([ ! -d monorepo ] && git clone https://github.com/ethereum-optimism/monorepo) || exit 0 + cd monorepo && git checkout $(cat ../.monorepo) + +# Updates the pinned version of the monorepo +update-monorepo: + [ ! -d monorepo ] && git clone https://github.com/ethereum-optimism/monorepo + cd monorepo && git rev-parse HEAD > ../.monorepo + # Run action tests for the client program on the native target action-tests test_name='Test_ProgramAction' *args='': #!/bin/bash @@ -52,49 +107,4 @@ action-tests test_name='Test_ProgramAction' *args='': # Clean the action tests directory clean-actions: - rm -rf monorepo/ - -# Lint the workspace for all available targets -alias la := lint-all -lint-all: lint-native lint-docs - -# Runs `cargo hack check` against the workspace -hack: - cargo hack check --feature-powerset --no-dev-deps - -# Fixes the formatting of the workspace -fmt-native-fix: - cargo +nightly fmt --all - -# Check the formatting of the workspace -fmt-native-check: - cargo +nightly fmt --all -- --check - -# Lint the workspace -lint-native: fmt-native-check lint-docs - cargo +nightly clippy --workspace --all --all-features --all-targets -- -D warnings - -# Lint the Rust documentation -lint-docs: - RUSTDOCFLAGS="-D warnings" cargo doc --all --no-deps --document-private-items - -# Test the Rust documentation -test-docs: - cargo test --doc --all --locked - -# Build the workspace for all available targets -build: build-native - -# Build for the native target -build-native *args='': - cargo build --workspace $@ - -# Clones and checks out the monorepo at the commit present in `.monorepo` -monorepo: - ([ ! -d monorepo ] && git clone https://github.com/ethereum-optimism/monorepo) || exit 0 - cd monorepo && git checkout $(cat ../.monorepo) - -# Updates the pinned version of the monorepo -update-monorepo: - [ ! -d monorepo ] && git clone https://github.com/ethereum-optimism/monorepo - cd monorepo && git rev-parse HEAD > ../.monorepo + rm -rf monorepo/ \ No newline at end of file