From 50ed04e1683324d84423e36c05b07f848a852816 Mon Sep 17 00:00:00 2001 From: Junha Park <0xjunha@gmail.com> Date: Fri, 29 Nov 2024 17:18:57 +0900 Subject: [PATCH] [cli] Make chunk size configurable for chunked publish * Added a new `chunk_size` option to `ChunkedPublishOption` * Updated `MAX_CHUNK_SIZE_IN_BYTES` to be of type `&str` with a value of `55000` --- .../src/tests/large_package_publishing.rs | 4 +++- aptos-move/framework/src/chunked_publish.rs | 15 ++++++++------- crates/aptos/src/common/types.rs | 12 ++++++++++-- crates/aptos/src/move_tool/mod.rs | 9 +++++++++ crates/aptos/src/test/mod.rs | 3 ++- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/aptos-move/e2e-move-tests/src/tests/large_package_publishing.rs b/aptos-move/e2e-move-tests/src/tests/large_package_publishing.rs index 159dcc6ceb1b3..a450c62b9dd0c 100644 --- a/aptos-move/e2e-move-tests/src/tests/large_package_publishing.rs +++ b/aptos-move/e2e-move-tests/src/tests/large_package_publishing.rs @@ -4,7 +4,8 @@ use crate::{assert_move_abort, assert_success, assert_vm_status, tests::common, MoveHarness}; use aptos_framework::{ chunked_publish::{ - chunk_package_and_create_payloads, PublishType, LARGE_PACKAGES_MODULE_ADDRESS, + chunk_package_and_create_payloads, PublishType, CHUNK_SIZE_IN_BYTES, + LARGE_PACKAGES_MODULE_ADDRESS, }, natives::{ code::{PackageMetadata, PackageRegistry, UpgradePolicy}, @@ -144,6 +145,7 @@ impl LargePackageTestContext { publish_type, Some(self.object_address), AccountAddress::from_str(LARGE_PACKAGES_MODULE_ADDRESS).unwrap(), + CHUNK_SIZE_IN_BYTES.parse().unwrap(), ) } } diff --git a/aptos-move/framework/src/chunked_publish.rs b/aptos-move/framework/src/chunked_publish.rs index 8d64d5b591b8f..b756fa7a25dd7 100644 --- a/aptos-move/framework/src/chunked_publish.rs +++ b/aptos-move/framework/src/chunked_publish.rs @@ -9,8 +9,8 @@ use move_core_types::{account_address::AccountAddress, ident_str, language_stora pub const LARGE_PACKAGES_MODULE_ADDRESS: &str = "0x0e1ca3011bdd07246d4d16d909dbb2d6953a86c4735d5acf5865d962c630cce7"; -/// Maximum code & metadata chunk size to be included in a transaction -pub const MAX_CHUNK_SIZE_IN_BYTES: usize = 60_000; +/// The default chunk size for splitting code and metadata to fit within the transaction size limits. +pub const CHUNK_SIZE_IN_BYTES: &str = "55000"; pub enum PublishType { AccountDeploy, @@ -24,9 +24,10 @@ pub fn chunk_package_and_create_payloads( publish_type: PublishType, object_address: Option, large_packages_module_address: AccountAddress, + chunk_size: usize, ) -> Vec { // Chunk the metadata - let mut metadata_chunks = create_chunks(metadata); + let mut metadata_chunks = create_chunks(metadata, chunk_size); // Separate last chunk for special handling let mut metadata_chunk = metadata_chunks.pop().expect("Metadata is required"); @@ -42,9 +43,9 @@ pub fn chunk_package_and_create_payloads( let mut code_chunks: Vec> = vec![]; for (idx, module_code) in package_code.into_iter().enumerate() { - let chunked_module = create_chunks(module_code); + let chunked_module = create_chunks(module_code, chunk_size); for chunk in chunked_module { - if taken_size + chunk.len() > MAX_CHUNK_SIZE_IN_BYTES { + if taken_size + chunk.len() > chunk_size { // Create a payload and reset accumulators let payload = large_packages_stage_code_chunk( metadata_chunk, @@ -94,8 +95,8 @@ pub fn chunk_package_and_create_payloads( } // Create chunks of data based on the defined maximum chunk size. -fn create_chunks(data: Vec) -> Vec> { - data.chunks(MAX_CHUNK_SIZE_IN_BYTES) +fn create_chunks(data: Vec, chunk_size: usize) -> Vec> { + data.chunks(chunk_size) .map(|chunk| chunk.to_vec()) .collect() } diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index db0673f6d0efd..a4393eff4dd6b 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -25,7 +25,7 @@ use aptos_crypto::{ encoding_type::{EncodingError, EncodingType}, x25519, PrivateKey, ValidCryptoMaterialStringExt, }; -use aptos_framework::chunked_publish::LARGE_PACKAGES_MODULE_ADDRESS; +use aptos_framework::chunked_publish::{CHUNK_SIZE_IN_BYTES, LARGE_PACKAGES_MODULE_ADDRESS}; use aptos_global_constants::adjust_gas_headroom; use aptos_keygen::KeyGen; use aptos_logger::Level; @@ -2370,8 +2370,16 @@ pub struct ChunkedPublishOption { /// Address of the `large_packages` move module for chunked publishing /// /// By default, on the module is published at `0x0e1ca3011bdd07246d4d16d909dbb2d6953a86c4735d5acf5865d962c630cce7` - /// on Testnet and Mainnet. On any other network, you will need to first publish it from the framework + /// on Testnet and Mainnet. On any other network, you will need to first publish it from the framework /// under move-examples/large_packages. #[clap(long, default_value = LARGE_PACKAGES_MODULE_ADDRESS, value_parser = crate::common::types::load_account_arg)] pub(crate) large_packages_module_address: AccountAddress, + + /// Size of the code chunk in bytes for splitting bytecode and metadata of large packages + /// + /// By default, the chunk size is set to `CHUNK_SIZE_IN_BYTES`. A smaller chunk size will result + /// in more transactions required to publish a package, while a larger chunk size might cause + /// transaction to fail due to exceeding the execution gas limit. + #[clap(long, default_value = CHUNK_SIZE_IN_BYTES, value_parser = clap::value_parser!(usize))] + pub(crate) chunk_size: usize, } diff --git a/crates/aptos/src/move_tool/mod.rs b/crates/aptos/src/move_tool/mod.rs index bd0da93576ca6..b2959e013c0b8 100644 --- a/crates/aptos/src/move_tool/mod.rs +++ b/crates/aptos/src/move_tool/mod.rs @@ -821,6 +821,7 @@ impl AsyncTryInto for &PublishPackage { PublishType::AccountDeploy, None, self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )?; let size = &chunked_publish_payloads @@ -1013,6 +1014,7 @@ fn create_chunked_publish_payloads( publish_type: PublishType, object_address: Option, large_packages_module_address: AccountAddress, + chunk_size: usize, ) -> CliTypedResult { let compiled_units = package.extract_code(); let metadata = package.extract_metadata()?; @@ -1030,6 +1032,7 @@ fn create_chunked_publish_payloads( publish_type, maybe_object_address, large_packages_module_address, + chunk_size, ); Ok(ChunkedPublishPayloads { payloads }) @@ -1157,6 +1160,7 @@ impl CliCommand for CreateObjectAndPublishPackage { PublishType::AccountDeploy, None, self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; let staging_tx_count = (mock_payloads.len() - 1) as u64; @@ -1183,6 +1187,7 @@ impl CliCommand for CreateObjectAndPublishPackage { PublishType::ObjectDeploy, None, self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; @@ -1295,6 +1300,7 @@ impl CliCommand for UpgradeObjectPackage { PublishType::ObjectUpgrade, Some(self.object_address), self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; @@ -1386,6 +1392,7 @@ impl CliCommand for DeployObjectCode { PublishType::AccountDeploy, None, self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; let staging_tx_count = (mock_payloads.len() - 1) as u64; @@ -1412,6 +1419,7 @@ impl CliCommand for DeployObjectCode { PublishType::ObjectDeploy, None, self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; @@ -1530,6 +1538,7 @@ impl CliCommand for UpgradeCodeObject { PublishType::ObjectUpgrade, Some(self.object_address), self.chunked_publish_option.large_packages_module_address, + self.chunked_publish_option.chunk_size, )? .payloads; diff --git a/crates/aptos/src/test/mod.rs b/crates/aptos/src/test/mod.rs index c92fa415c0bd9..f0386d51c3387 100644 --- a/crates/aptos/src/test/mod.rs +++ b/crates/aptos/src/test/mod.rs @@ -53,7 +53,7 @@ use aptos_crypto::{ ed25519::{Ed25519PrivateKey, Ed25519PublicKey}, x25519, PrivateKey, }; -use aptos_framework::chunked_publish::LARGE_PACKAGES_MODULE_ADDRESS; +use aptos_framework::chunked_publish::{CHUNK_SIZE_IN_BYTES, LARGE_PACKAGES_MODULE_ADDRESS}; use aptos_genesis::config::HostAndPort; use aptos_keygen::KeyGen; use aptos_logger::warn; @@ -900,6 +900,7 @@ impl CliTestFramework { LARGE_PACKAGES_MODULE_ADDRESS, ) .unwrap(), + chunk_size: CHUNK_SIZE_IN_BYTES.parse().unwrap(), }, } .execute()