Feat : removed init_config function from tests (#87)

* feat : removed init_config function from tests

* Feat: proving job storage integration (#88)

* feat : added storage client fetch for pie file

* feat : added error report in function body

* feat : added errors

* feat : added s3 error type in proving job process job function

* feat : updated code

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Added
 
+- support for fetching PIE file from storage client in proving job.
 - added coveralls support
 - moved mongodb serde behind feature flag
 - implemented DA worker.
@@ -32,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Removed
 
+- `init_config` from all the tests.
 - `fetch_from_test` argument
 
 ## Fixed

crates/orchestrator/src/jobs/constants.rs

@@ -1,9 +1,5 @@
 pub const JOB_PROCESS_ATTEMPT_METADATA_KEY: &str = "process_attempt_no";
-
 pub const JOB_VERIFICATION_ATTEMPT_METADATA_KEY: &str = "verification_attempt_no";
-
-pub const JOB_METADATA_CAIRO_PIE_PATH_KEY: &str = "cairo_pie_path";
-
 pub const JOB_METADATA_STATE_UPDATE_BLOCKS_TO_SETTLE_KEY: &str = "blocks_number_to_settle";
 pub const JOB_METADATA_STATE_UPDATE_FETCH_FROM_TESTS: &str = "fetch_from_test_data";
 pub const JOB_METADATA_STATE_UPDATE_ATTEMPT_PREFIX: &str = "attempt_tx_hashes_";

crates/orchestrator/src/jobs/da_job/mod.rs

@@ -374,8 +374,11 @@ pub mod test {
     use std::fs;
     use std::fs::File;
     use std::io::Read;
+    use std::sync::Arc;
 
+    use crate::config::config;
     use crate::data_storage::MockDataStorage;
+    use crate::tests::config::TestConfigBuilder;
     use ::serde::{Deserialize, Serialize};
     use color_eyre::Result;
     use da_client_interface::MockDaClient;
@@ -385,9 +388,10 @@ pub mod test {
     use majin_blob_types::state_diffs::UnorderedEq;
     use rstest::rstest;
     use serde_json::json;
+    use starknet::providers::jsonrpc::HttpTransport;
+    use starknet::providers::JsonRpcClient;
     use starknet_core::types::{FieldElement, StateUpdate};
-
-    use crate::tests::common::init_config;
+    use url::Url;
 
     /// Tests `da_word` function with various inputs for class flag, new nonce, and number of changes.
     /// Verifies that `da_word` produces the correct FieldElement based on the provided parameters.
@@ -453,16 +457,19 @@ pub mod test {
         // Mocking storage client
         storage_client.expect_put_data().returning(|_, _| Result::Ok(())).times(1);
 
-        let config = init_config(
-            Some(format!("http://localhost:{}", server.port())),
-            None,
-            None,
-            Some(da_client),
-            None,
-            None,
-            Some(storage_client),
-        )
-        .await;
+        let provider = JsonRpcClient::new(HttpTransport::new(
+            Url::parse(format!("http://localhost:{}", server.port()).as_str()).expect("Failed to parse URL"),
+        ));
+
+        // mock block number (madara) : 5
+        TestConfigBuilder::new()
+            .mock_starknet_client(Arc::new(provider))
+            .mock_da_client(Box::new(da_client))
+            .mock_storage_client(Box::new(storage_client))
+            .build()
+            .await;
+
+        let config = config().await;
 
         get_nonce_attached(&server, nonce_file_path);
 

crates/orchestrator/src/jobs/proving_job/mod.rs

@@ -1,6 +1,4 @@
 use std::collections::HashMap;
-use std::path::PathBuf;
-use std::str::FromStr;
 
 use async_trait::async_trait;
 use cairo_vm::vm::runners::cairo_pie::CairoPie;
@@ -11,7 +9,6 @@ use tracing::log::log;
 use tracing::log::Level::Error;
 use uuid::Uuid;
 
-use super::constants::JOB_METADATA_CAIRO_PIE_PATH_KEY;
 use super::types::{JobItem, JobStatus, JobType, JobVerificationStatus};
 use super::{Job, JobError, OtherError};
 use crate::config::Config;
@@ -22,7 +19,10 @@ pub enum ProvingError {
     CairoPIEWrongPath { internal_id: String },
 
     #[error("Not able to read the cairo PIE file from the zip file provided.")]
-    CairoPIENotReadable,
+    CairoPIENotReadable(String),
+
+    #[error("Not able to get the PIE file from AWS S3 bucket.")]
+    CairoPIEFileFetchFailed(String),
 
     #[error("Other error: {0}")]
     Other(#[from] OtherError),
@@ -38,10 +38,6 @@ impl Job for ProvingJob {
         internal_id: String,
         metadata: HashMap<String, String>,
     ) -> Result<JobItem, JobError> {
-        if !metadata.contains_key(JOB_METADATA_CAIRO_PIE_PATH_KEY) {
-            // TODO: validate the usage of `.clone()` here, ensure lightweight borrowing of variables
-            Err(ProvingError::CairoPIEWrongPath { internal_id: internal_id.clone() })?
-        }
         Ok(JobItem {
             id: Uuid::new_v4(),
             internal_id,
@@ -54,15 +50,15 @@ impl Job for ProvingJob {
     }
 
     async fn process_job(&self, config: &Config, job: &mut JobItem) -> Result<String, JobError> {
-        // TODO: allow to download PIE from storage
-        let cairo_pie_path = job
-            .metadata
-            .get(JOB_METADATA_CAIRO_PIE_PATH_KEY)
-            .map(|s| PathBuf::from_str(s))
-            .ok_or_else(|| ProvingError::CairoPIEWrongPath { internal_id: job.internal_id.clone() })?
-            .map_err(|_| ProvingError::CairoPIENotReadable)?;
-
-        let cairo_pie = CairoPie::read_zip_file(&cairo_pie_path).map_err(|_| ProvingError::CairoPIENotReadable)?;
+        // Cairo Pie path in s3 storage client
+        let cairo_pie_path = job.internal_id.to_string() + "/pie.zip";
+        let cairo_pie_file = config
+            .storage()
+            .get_data(&cairo_pie_path)
+            .await
+            .map_err(|e| ProvingError::CairoPIEFileFetchFailed(e.to_string()))?;
+        let cairo_pie = CairoPie::from_bytes(cairo_pie_file.to_vec().as_slice())
+            .map_err(|e| ProvingError::CairoPIENotReadable(e.to_string()))?;
 
         let external_id = config
             .prover_client()

crates/orchestrator/src/tests/common/mod.rs

@@ -1,66 +1,23 @@
 pub mod constants;
 
 use std::collections::HashMap;
-use std::sync::Arc;
 
 use ::uuid::Uuid;
 use aws_config::Region;
-use constants::*;
-use da_client_interface::MockDaClient;
 use mongodb::Client;
-use prover_client_interface::MockProverClient;
 use rstest::*;
 use serde::Deserialize;
-use settlement_client_interface::MockSettlementClient;
-use starknet::providers::jsonrpc::HttpTransport;
-use starknet::providers::JsonRpcClient;
-use url::Url;
 
-use crate::config::Config;
 use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig};
 use crate::data_storage::aws_s3::AWSS3;
-use crate::data_storage::{DataStorage, DataStorageConfig, MockDataStorage};
+use crate::data_storage::{DataStorage, DataStorageConfig};
 use crate::database::mongodb::config::MongoDbConfig;
 use crate::database::mongodb::MongoDb;
-use crate::database::{DatabaseConfig, MockDatabase};
+use crate::database::DatabaseConfig;
 use crate::jobs::types::JobStatus::Created;
 use crate::jobs::types::JobType::DataSubmission;
 use crate::jobs::types::{ExternalId, JobItem};
 use crate::queue::job_queue::{JOB_PROCESSING_QUEUE, JOB_VERIFICATION_QUEUE};
-use crate::queue::MockQueueProvider;
-
-pub async fn init_config(
-    rpc_url: Option<String>,
-    database: Option<MockDatabase>,
-    queue: Option<MockQueueProvider>,
-    da_client: Option<MockDaClient>,
-    prover_client: Option<MockProverClient>,
-    settlement_client: Option<MockSettlementClient>,
-    storage_client: Option<MockDataStorage>,
-) -> Config {
-    let _ = tracing_subscriber::fmt().with_max_level(tracing::Level::INFO).with_target(false).try_init();
-
-    let rpc_url = rpc_url.unwrap_or(MADARA_RPC_URL.to_string());
-    let database = database.unwrap_or_default();
-    let queue = queue.unwrap_or_default();
-    let da_client = da_client.unwrap_or_default();
-    let prover_client = prover_client.unwrap_or_default();
-    let settlement_client = settlement_client.unwrap_or_default();
-    let storage_client = storage_client.unwrap_or_default();
-
-    // init starknet client
-    let provider = JsonRpcClient::new(HttpTransport::new(Url::parse(rpc_url.as_str()).expect("Failed to parse URL")));
-
-    Config::new(
-        Arc::new(provider),
-        Box::new(da_client),
-        Box::new(prover_client),
-        Box::new(settlement_client),
-        Box::new(database),
-        Box::new(queue),
-        Box::new(storage_client),
-    )
-}
 
 #[fixture]
 pub fn default_job_item() -> JobItem {

crates/orchestrator/src/tests/config.rs

@@ -76,6 +76,26 @@ impl TestConfigBuilder {
         self
     }
 
+    pub fn mock_starknet_client(mut self, starknet_client: Arc<JsonRpcClient<HttpTransport>>) -> TestConfigBuilder {
+        self.starknet_client = Some(starknet_client);
+        self
+    }
+
+    pub fn mock_prover_client(mut self, prover_client: Box<dyn ProverClient>) -> TestConfigBuilder {
+        self.prover_client = Some(prover_client);
+        self
+    }
+
+    pub fn mock_storage_client(mut self, storage_client: Box<dyn DataStorage>) -> TestConfigBuilder {
+        self.storage = Some(storage_client);
+        self
+    }
+
+    pub fn mock_queue(mut self, queue: Box<dyn QueueProvider>) -> TestConfigBuilder {
+        self.queue = Some(queue);
+        self
+    }
+
     pub async fn build(mut self) -> MockServer {
         dotenvy::from_filename("../.env.test").expect("Failed to load the .env file");
 

crates/orchestrator/src/tests/jobs/proving_job/mod.rs

@@ -1,28 +1,34 @@
+use bytes::Bytes;
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::path::Path;
+use std::sync::Arc;
 
-use crate::config::{config, config_force_init};
+use crate::config::config;
+use crate::data_storage::MockDataStorage;
 use httpmock::prelude::*;
+use mockall::predicate::eq;
 use prover_client_interface::{MockProverClient, TaskStatus};
 use rstest::*;
+use starknet::providers::jsonrpc::HttpTransport;
+use starknet::providers::JsonRpcClient;
+use url::Url;
 use uuid::Uuid;
 
-use super::super::common::{default_job_item, init_config};
-use crate::jobs::constants::JOB_METADATA_CAIRO_PIE_PATH_KEY;
+use super::super::common::default_job_item;
 use crate::jobs::proving_job::ProvingJob;
 use crate::jobs::types::{JobItem, JobStatus, JobType};
 use crate::jobs::Job;
+use crate::tests::config::TestConfigBuilder;
 
 #[rstest]
 #[tokio::test]
 async fn test_create_job() {
-    let config = init_config(None, None, None, None, None, None, None).await;
-    let job = ProvingJob
-        .create_job(
-            &config,
-            String::from("0"),
-            HashMap::from([(JOB_METADATA_CAIRO_PIE_PATH_KEY.into(), "pie.zip".into())]),
-        )
-        .await;
+    TestConfigBuilder::new().build().await;
+    let config = config().await;
+
+    let job = ProvingJob.create_job(&config, String::from("0"), HashMap::new()).await;
     assert!(job.is_ok());
 
     let job = job.unwrap();
@@ -41,32 +47,38 @@ async fn test_verify_job(#[from(default_job_item)] mut job_item: JobItem) {
     let mut prover_client = MockProverClient::new();
     prover_client.expect_get_task_status().times(1).returning(|_| Ok(TaskStatus::Succeeded));
 
-    let config = init_config(None, None, None, None, Some(prover_client), None, None).await;
+    TestConfigBuilder::new().mock_prover_client(Box::new(prover_client)).build().await;
+
+    let config = config().await;
     assert!(ProvingJob.verify_job(&config, &mut job_item).await.is_ok());
 }
 
 #[rstest]
 #[tokio::test]
 async fn test_process_job() {
     let server = MockServer::start();
-
     let mut prover_client = MockProverClient::new();
+
     prover_client.expect_submit_task().times(1).returning(|_| Ok("task_id".to_string()));
+    let provider = JsonRpcClient::new(HttpTransport::new(
+        Url::parse(format!("http://localhost:{}", server.port()).as_str()).expect("Failed to parse URL"),
+    ));
 
-    let config_init = init_config(
-        Some(format!("http://localhost:{}", server.port())),
-        None,
-        None,
-        None,
-        Some(prover_client),
-        None,
-        None,
-    )
-    .await;
+    let mut file =
+        File::open(Path::new(&format!("{}/src/tests/artifacts/fibonacci.zip", env!("CARGO_MANIFEST_DIR")))).unwrap();
+    let mut buffer = Vec::new();
+    file.read_to_end(&mut buffer).unwrap();
 
-    config_force_init(config_init).await;
+    let mut storage = MockDataStorage::new();
+    let buffer_bytes = Bytes::from(buffer);
+    storage.expect_get_data().with(eq("0/pie.zip")).return_once(move |_| Ok(buffer_bytes));
 
-    let cairo_pie_path = format!("{}/src/tests/artifacts/fibonacci.zip", env!("CARGO_MANIFEST_DIR"));
+    TestConfigBuilder::new()
+        .mock_starknet_client(Arc::new(provider))
+        .mock_prover_client(Box::new(prover_client))
+        .mock_storage_client(Box::new(storage))
+        .build()
+        .await;
 
     assert_eq!(
         ProvingJob
@@ -78,7 +90,7 @@ async fn test_process_job() {
                     job_type: JobType::ProofCreation,
                     status: JobStatus::Created,
                     external_id: String::new().into(),
-                    metadata: HashMap::from([(JOB_METADATA_CAIRO_PIE_PATH_KEY.into(), cairo_pie_path)]),
+                    metadata: HashMap::new(),
                     version: 0,
                 }
             )