feat(core,ffi): Add Swift tests for MSM benchmarking

mopro-core/Cargo.toml

@@ -13,7 +13,7 @@ wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" }
 [features]
 default = ["wasmer/dylib"]
 dylib = []                                        # NOTE: can probably remove this if we use env config instead
-gpu-benchmarks = ["jemalloc-ctl", "jemallocator"]
+gpu-benchmarks = []
 
 [dependencies]
 ark-circom = { git = "https://github.com/arkworks-rs/circom-compat.git" }
@@ -46,10 +46,6 @@ thiserror = "=1.0.39"
 color-eyre = "=0.6.2"
 criterion = "=0.3.6"
 
-# GPU benchmarks
-jemalloc-ctl = { version = "0.5.4", optional = true }
-jemallocator = { version = "0.5.4", optional = true }
-
 [build-dependencies]
 color-eyre = "0.6"
 enumset = "1.0.8"

mopro-core/README.md

@@ -45,4 +45,10 @@ Note that `APPLE_SIGNING_IDENTITY` must be set.
 
 ## To use ark-zkey
 
-Experimental support for significantly faster zkey loading. See `../ark-zkey` README for how to build arkzkey.
+Experimental support for significantly faster zkey loading. See `../ark-zkey` README for how to build arkzkey.
+
+## To run msm benchmark report on laptop
+
+`cargo run --release --features gpu-benchmarks --package mopro-core --bin generate_benchmark_report`
+
+The report will be in `mopro-core/benchmarks/gpu_explorations/msm_bench_rust_laptop.csv`.

mopro-core/benchmarks/gpu_explorations/msm_bench_rust_laptop.csv

@@ -0,0 +1,22 @@
+num_msm,avg_processing_time(ms),total_processing_time(ms)
+1,1.224375,1.224375
+500,0.36152366,180.76183
+1000,0.305950039,305.950039
+1500,0.3034417346666667,455.16260200000005
+2000,0.309008472,618.016944
+2500,0.3073222012,768.305503
+3000,0.3040002196666667,912.000659
+3500,0.30619671200000004,1071.688492
+4000,0.3045648925,1218.2595700000002
+4500,0.3030858693333333,1363.886412
+5000,0.31226579120000003,1561.328956
+5500,0.3099467858181818,1704.707322
+6000,0.307641547,1845.8492820000001
+6500,0.30413866584615384,1976.901328
+7000,0.3078945885714286,2155.2621200000003
+7500,0.3031438489333333,2273.5788669999997
+8000,0.303863230625,2430.9058449999998
+8500,0.3024968967058823,2571.223622
+9000,0.3026250855555555,2723.62577
+9500,0.30347056831578945,2882.970399
+10000,0.30400440209999996,3040.0440209999997

mopro-core/benchmarks/gpu_explorations/msm_bench_swift_laptop.csv

@@ -0,0 +1,22 @@
+num_msm,avg_processing_time(ms),total_processing_time(ms)
+1,0.956709,0.956709
+500,0.428701728,214.350864
+1000,0.580982619,580.982619
+1500,0.37609757466666666,564.1463620000001
+2000,0.363904182,727.8083640000001
+2500,0.3618204652,904.5511630000001
+3000,0.36476984566666665,1094.309537
+3500,0.36913610714285716,1291.976375
+4000,0.36055417300000003,1442.2166920000002
+4500,0.3586435595555556,1613.8960180000001
+5000,0.35959354720000003,1797.9677359999998
+5500,0.3742771358181818,2058.524247
+6000,0.3626761886666666,2176.057132
+6500,0.36803800384615387,2392.247025
+7000,0.371516067,2600.612469
+7500,0.3678911301333333,2759.183476
+8000,0.3686055185,2948.844148
+8500,0.4007246014117647,3406.1591120000003
+9000,0.3771045951111111,3393.941356
+9500,0.37394299821052635,3552.458483
+10000,0.3638388068,3638.388068

mopro-core/src/middleware/gpu_explorations/bin/generate_benchmark_report.rs

@@ -10,11 +10,11 @@ use {
 fn main() {
     let path = env::current_dir()
         .unwrap()
-        .join("benchmarks/gpu_explorations/msm_bench.csv");
-    let mut file = File::create(path).unwrap();
+        .join("benchmarks/gpu_explorations/msm_bench_rust_laptop.csv");
+    let mut file = File::create(path.clone()).unwrap();
     writeln!(
         file,
-        "num_msm,avg_processing_time(ms),total_processing_time(ms),memory_allocated(Bytes)"
+        "num_msm,avg_processing_time(ms),total_processing_time(ms)"
     )
     .unwrap();
     // generate trials = [1, 500, 1_000, 1_500, ..., 10_000]
@@ -23,14 +23,12 @@ fn main() {
         let bench_data = run_msm_benchmark(Some(each)).unwrap();
         writeln!(
             file,
-            "{},{},{},{}",
-            bench_data.num_msm,
-            bench_data.avg_processing_time,
-            bench_data.total_processing_time,
-            bench_data.allocated_memory,
+            "{},{},{}",
+            bench_data.num_msm, bench_data.avg_processing_time, bench_data.total_processing_time,
         )
         .unwrap();
     }
+    println!("Report generated at {:?}", path);
 }
 
 #[cfg(not(feature = "gpu-benchmarks"))]

mopro-core/src/middleware/gpu_explorations/mod.rs

@@ -3,15 +3,13 @@ use ark_ec::VariableBaseMSM;
 use ark_std::{error::Error, UniformRand};
 
 // For benchmarking
-use jemalloc_ctl::{epoch, stats};
 use std::time::{Duration, Instant};
 
 #[derive(Debug, Clone)]
 pub struct BenchmarkResult {
     pub num_msm: u32,
     pub avg_processing_time: f64,
     pub total_processing_time: f64,
-    pub allocated_memory: u32,
 }
 
 fn single_msm() -> Result<(), Box<dyn Error>> {
@@ -33,26 +31,19 @@ fn single_msm() -> Result<(), Box<dyn Error>> {
 pub fn run_msm_benchmark(num_msm: Option<u32>) -> Result<BenchmarkResult, Box<dyn Error>> {
     let num_msm = num_msm.unwrap_or(1000); // default to 1000 msm operations
 
-    let mem_epoch = epoch::mib().unwrap(); // For updating jemalloc stats of memory usage
-    let allocated = stats::allocated::mib().unwrap();
-
     let mut total_msm = Duration::new(0, 0);
-
     for _ in 0..num_msm {
         let start = Instant::now();
         single_msm()?;
         total_msm += start.elapsed();
     }
-    mem_epoch.advance().unwrap(); // Update msm memory usage
 
-    let allocated_size = allocated.read().unwrap() as u32; // in Bytes
     let msm_avg = (total_msm.as_secs_f64() / num_msm as f64) * 1_000.0; // in ms
 
     Ok(BenchmarkResult {
         num_msm,
         avg_processing_time: msm_avg,
         total_processing_time: total_msm.as_secs_f64() * 1_000.0,
-        allocated_memory: allocated_size,
     })
 }
 
@@ -73,9 +64,5 @@ mod tests {
             "└─ Average msm time: {:.5} ms\n└─ Overall processing time: {:.5} ms",
             benchmarks.avg_processing_time, benchmarks.total_processing_time
         );
-        println!(
-            "└─ Memory allocated: {:.5} Bytes",
-            benchmarks.allocated_memory,
-        );
     }
 }

mopro-ffi/Cargo.toml

@@ -15,6 +15,7 @@ path = "uniffi-bindgen.rs"
 
 [features]
 default = []
+# default = ["gpu-benchmarks"]
 
 # If we enable them here, they should be enabled in mopro-core as well
 dylib = ["mopro-core/dylib"]

mopro-ffi/README.md

@@ -46,3 +46,15 @@ To test bindings:
 To test bindings in release mode without warning:
 
 `cargo test --test test_generated_bindings --release 2>/dev/null`
+
+## Generate MSM benchmark report in swift on laptop
+
+1. Comment `default=[]` and uncomment `default=["gpu-benchmarks"]` to enable `gpu-benchmarks` feature flag
+2. run `RUSTFLAGS="-C opt-level=3" cargo test --test test_generated_bindings --release`
+3. The report will be generated at `mopro-core/benchmarks/gpu_explorations/msm_bench_swift_laptop.csv`
+
+![msm benchmarks of time (rust vs. swift)](https://hackmd.io/_uploads/BkxRMMtca.png)
+
+The above report was conducted on an M1 Pro MacBook Pro.
+
+For more information, read this report we have completed, which compares the time performance of MSM running in rust and swift.

mopro-ffi/src/lib.rs

@@ -55,7 +55,6 @@ pub struct BenchmarkResult {
     pub num_msm: u32,
     pub avg_processing_time: f64,
     pub total_processing_time: f64,
-    pub allocated_memory: u32,
 }
 
 //     pub inputs: Vec<u8>,
@@ -226,7 +225,7 @@ pub fn run_msm_benchmark(num_msm: Option<u32>) -> Result<BenchmarkResult, MoproE
 }
 
 #[cfg(not(feature = "gpu-benchmarks"))]
-pub fn run_msm_benchmark(num_msm: Option<u32>) -> Result<BenchmarkResult, MoproError> {
+pub fn run_msm_benchmark(_num_msm: Option<u32>) -> Result<BenchmarkResult, MoproError> {
     println!("gpu-benchmarks feature not enabled!");
     panic!("gpu-benchmarks feature not enabled!");
 }
@@ -389,10 +388,6 @@ mod tests {
             "└─ Average msm time: {:.5} ms\n└─ Overall processing time: {:.5} ms",
             benchmarks.avg_processing_time, benchmarks.total_processing_time
         );
-        println!(
-            "└─ Memory allocated: {:.5} Bytes",
-            benchmarks.allocated_memory,
-        );
         Ok(())
     }
 }

mopro-ffi/src/mopro.udl

@@ -34,7 +34,6 @@ dictionary BenchmarkResult {
   u32 num_msm;
   double avg_processing_time;
   double total_processing_time;
-  u32 allocated_memory;
 };
 
 dictionary G1 {

mopro-ffi/tests/bindings/test_mopro_gen_benchmarks_report.swift

@@ -0,0 +1,26 @@
+import mopro
+import Foundation
+
+let path = FileManager.default.currentDirectoryPath + "/../../../../mopro-core/benchmarks/gpu_explorations/msm_bench_swift_laptop.csv"
+let fileURL = URL(fileURLWithPath: path)
+var fileContent = "num_msm,avg_processing_time(ms),total_processing_time(ms)\n"
+
+// generate trials = [1, 500, 1000, 1500, ... ,10000]
+let trials: [UInt32] = (0..<21).map { max($0 * 500, 1)}
+
+for each in trials {
+    do {
+        // for tracking the progress
+        print("Running benchmark with \(each) MSMs...")
+        let benchData: BenchmarkResult = try runMsmBenchmark(numMsm: each)
+        fileContent += "\(benchData.numMsm),\(benchData.avgProcessingTime),\(benchData.totalProcessingTime)\n"
+    } catch let error as MoproError{
+        print("Error running benchmark: \(error)")
+    }
+}
+
+do {
+    try fileContent.write(to: fileURL, atomically: true, encoding:.utf8)
+} catch {
+        print("Error writing file: \(error)")
+}

mopro-ffi/tests/bindings/test_mopro_gpu_benchmarks.swift

@@ -0,0 +1,11 @@
+import mopro
+import Foundation
+
+do {
+    let benchData: BenchmarkResult = try runMsmBenchmark(numMsm: 1000)
+    print("\nBenchmarking \(1000) msm on BN254 curve")
+    print("└─ Average msm time: \(benchData.avgProcessingTime) ms")
+    print("└─ Overall processing time: \(benchData.totalProcessingTime) ms")
+} catch let error as MoproError{
+    print("Error running benchmark: \(error)")
+}

mopro-ffi/tests/test_generated_bindings.rs

@@ -1,3 +1,10 @@
+#[cfg(feature = "gpu-benchmarks")]
+uniffi::build_foreign_language_testcases!(
+    "tests/bindings/test_mopro_gen_benchmarks_report.swift",
+    "tests/bindings/test_mopro_gpu_benchmarks.swift",
+);
+
+#[cfg(not(feature = "gpu-benchmarks"))]
 uniffi::build_foreign_language_testcases!(
     "tests/bindings/test_mopro.swift",
     "tests/bindings/test_mopro.kts",