[cli] Make chunk size configurable for chunked publish (aptos-labs#15427

)

* [cli] Make chunk size configurable for chunked publish

* Added a new `chunk_size` option to `ChunkedPublishOption`
* Updated `MAX_CHUNK_SIZE_IN_BYTES` to be 55,000
* Add edchangelog

aptos-move/e2e-move-tests/src/tests/large_package_publishing.rs

@@ -4,7 +4,8 @@
 use crate::{assert_move_abort, assert_success, assert_vm_status, tests::common, MoveHarness};
 use aptos_framework::{
     chunked_publish::{
-        chunk_package_and_create_payloads, PublishType, LARGE_PACKAGES_MODULE_ADDRESS,
+        chunk_package_and_create_payloads, PublishType, CHUNK_SIZE_IN_BYTES,
+        LARGE_PACKAGES_MODULE_ADDRESS,
     },
     natives::{
         code::{PackageMetadata, PackageRegistry, UpgradePolicy},
@@ -144,6 +145,7 @@ impl LargePackageTestContext {
             publish_type,
             Some(self.object_address),
             AccountAddress::from_str(LARGE_PACKAGES_MODULE_ADDRESS).unwrap(),
+            CHUNK_SIZE_IN_BYTES,
         )
     }
 }

aptos-move/framework/src/chunked_publish.rs

@@ -9,8 +9,8 @@ use move_core_types::{account_address::AccountAddress, ident_str, language_stora
 pub const LARGE_PACKAGES_MODULE_ADDRESS: &str =
     "0x0e1ca3011bdd07246d4d16d909dbb2d6953a86c4735d5acf5865d962c630cce7";
 
-/// Maximum code & metadata chunk size to be included in a transaction
-pub const MAX_CHUNK_SIZE_IN_BYTES: usize = 60_000;
+/// The default chunk size for splitting code and metadata to fit within the transaction size limits.
+pub const CHUNK_SIZE_IN_BYTES: usize = 55_000;
 
 pub enum PublishType {
     AccountDeploy,
@@ -24,9 +24,10 @@ pub fn chunk_package_and_create_payloads(
     publish_type: PublishType,
     object_address: Option<AccountAddress>,
     large_packages_module_address: AccountAddress,
+    chunk_size: usize,
 ) -> Vec<TransactionPayload> {
     // Chunk the metadata
-    let mut metadata_chunks = create_chunks(metadata);
+    let mut metadata_chunks = create_chunks(metadata, chunk_size);
     // Separate last chunk for special handling
     let mut metadata_chunk = metadata_chunks.pop().expect("Metadata is required");
 
@@ -42,9 +43,9 @@ pub fn chunk_package_and_create_payloads(
     let mut code_chunks: Vec<Vec<u8>> = vec![];
 
     for (idx, module_code) in package_code.into_iter().enumerate() {
-        let chunked_module = create_chunks(module_code);
+        let chunked_module = create_chunks(module_code, chunk_size);
         for chunk in chunked_module {
-            if taken_size + chunk.len() > MAX_CHUNK_SIZE_IN_BYTES {
+            if taken_size + chunk.len() > chunk_size {
                 // Create a payload and reset accumulators
                 let payload = large_packages_stage_code_chunk(
                     metadata_chunk,
@@ -94,8 +95,8 @@ pub fn chunk_package_and_create_payloads(
 }
 
 // Create chunks of data based on the defined maximum chunk size.
-fn create_chunks(data: Vec<u8>) -> Vec<Vec<u8>> {
-    data.chunks(MAX_CHUNK_SIZE_IN_BYTES)
+fn create_chunks(data: Vec<u8>, chunk_size: usize) -> Vec<Vec<u8>> {
+    data.chunks(chunk_size)
         .map(|chunk| chunk.to_vec())
         .collect()
 }

crates/aptos/CHANGELOG.md

@@ -5,7 +5,8 @@ All notable changes to the Aptos CLI will be captured in this file. This project
 # Unreleased
 
 - Add `--node-api-key` flag to `aptos move replay` to allow for querying the fullnode with an API key.
-
+- Add `--chunk-size` flag to allow configuring chunk size for chunked publish mode.
+- Update the default chunk size for chunked publish mode (`CHUNK_SIZE_IN_BYTES`) from 60,000 to 55,000.
 
 ## [4.5.0] - 2024/11/15
 - Determine network from URL to make explorer links better for legacy users

crates/aptos/src/common/types.rs

@@ -25,7 +25,7 @@ use aptos_crypto::{
     encoding_type::{EncodingError, EncodingType},
     x25519, PrivateKey, ValidCryptoMaterialStringExt,
 };
-use aptos_framework::chunked_publish::LARGE_PACKAGES_MODULE_ADDRESS;
+use aptos_framework::chunked_publish::{CHUNK_SIZE_IN_BYTES, LARGE_PACKAGES_MODULE_ADDRESS};
 use aptos_global_constants::adjust_gas_headroom;
 use aptos_keygen::KeyGen;
 use aptos_logger::Level;
@@ -2370,8 +2370,16 @@ pub struct ChunkedPublishOption {
     /// Address of the `large_packages` move module for chunked publishing
     ///
     /// By default, on the module is published at `0x0e1ca3011bdd07246d4d16d909dbb2d6953a86c4735d5acf5865d962c630cce7`
-    /// on Testnet and Mainnet.  On any other network, you will need to first publish it from the framework
+    /// on Testnet and Mainnet. On any other network, you will need to first publish it from the framework
     /// under move-examples/large_packages.
     #[clap(long, default_value = LARGE_PACKAGES_MODULE_ADDRESS, value_parser = crate::common::types::load_account_arg)]
     pub(crate) large_packages_module_address: AccountAddress,
+
+    /// Size of the code chunk in bytes for splitting bytecode and metadata of large packages
+    ///
+    /// By default, the chunk size is set to `CHUNK_SIZE_IN_BYTES`. A smaller chunk size will result
+    /// in more transactions required to publish a package, while a larger chunk size might cause
+    /// transaction to fail due to exceeding the execution gas limit.
+    #[clap(long, default_value_t = CHUNK_SIZE_IN_BYTES)]
+    pub(crate) chunk_size: usize,
 }

crates/aptos/src/move_tool/mod.rs

@@ -821,6 +821,7 @@ impl AsyncTryInto<ChunkedPublishPayloads> for &PublishPackage {
             PublishType::AccountDeploy,
             None,
             self.chunked_publish_option.large_packages_module_address,
+            self.chunked_publish_option.chunk_size,
         )?;
 
         let size = &chunked_publish_payloads
@@ -1013,6 +1014,7 @@ fn create_chunked_publish_payloads(
     publish_type: PublishType,
     object_address: Option<AccountAddress>,
     large_packages_module_address: AccountAddress,
+    chunk_size: usize,
 ) -> CliTypedResult<ChunkedPublishPayloads> {
     let compiled_units = package.extract_code();
     let metadata = package.extract_metadata()?;
@@ -1030,6 +1032,7 @@ fn create_chunked_publish_payloads(
         publish_type,
         maybe_object_address,
         large_packages_module_address,
+        chunk_size,
     );
 
     Ok(ChunkedPublishPayloads { payloads })
@@ -1157,6 +1160,7 @@ impl CliCommand<TransactionSummary> for CreateObjectAndPublishPackage {
                 PublishType::AccountDeploy,
                 None,
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
             let staging_tx_count = (mock_payloads.len() - 1) as u64;
@@ -1183,6 +1187,7 @@ impl CliCommand<TransactionSummary> for CreateObjectAndPublishPackage {
                 PublishType::ObjectDeploy,
                 None,
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
 
@@ -1295,6 +1300,7 @@ impl CliCommand<TransactionSummary> for UpgradeObjectPackage {
                 PublishType::ObjectUpgrade,
                 Some(self.object_address),
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
 
@@ -1386,6 +1392,7 @@ impl CliCommand<TransactionSummary> for DeployObjectCode {
                 PublishType::AccountDeploy,
                 None,
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
             let staging_tx_count = (mock_payloads.len() - 1) as u64;
@@ -1412,6 +1419,7 @@ impl CliCommand<TransactionSummary> for DeployObjectCode {
                 PublishType::ObjectDeploy,
                 None,
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
 
@@ -1530,6 +1538,7 @@ impl CliCommand<TransactionSummary> for UpgradeCodeObject {
                 PublishType::ObjectUpgrade,
                 Some(self.object_address),
                 self.chunked_publish_option.large_packages_module_address,
+                self.chunked_publish_option.chunk_size,
             )?
             .payloads;
 

crates/aptos/src/test/mod.rs

@@ -53,7 +53,7 @@ use aptos_crypto::{
     ed25519::{Ed25519PrivateKey, Ed25519PublicKey},
     x25519, PrivateKey,
 };
-use aptos_framework::chunked_publish::LARGE_PACKAGES_MODULE_ADDRESS;
+use aptos_framework::chunked_publish::{CHUNK_SIZE_IN_BYTES, LARGE_PACKAGES_MODULE_ADDRESS};
 use aptos_genesis::config::HostAndPort;
 use aptos_keygen::KeyGen;
 use aptos_logger::warn;
@@ -900,6 +900,7 @@ impl CliTestFramework {
                     LARGE_PACKAGES_MODULE_ADDRESS,
                 )
                 .unwrap(),
+                chunk_size: CHUNK_SIZE_IN_BYTES,
             },
         }
         .execute()