From c7f0c2d1759eab0b3be4facc3f31537d8ac01f98 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sat, 10 Aug 2024 18:04:33 +0800 Subject: [PATCH] chore: add codspeed for benchmark --- .github/workflows/test_benchmark.yml | 56 ++++++++++++++++++++++++++ core/Cargo.lock | 35 ++++++++++++++++ core/Cargo.toml | 1 + core/benches/ops/main.rs | 4 +- core/benches/ops/read.rs | 7 ++-- core/benches/ops/write.rs | 2 +- core/benches/types/buffer.rs | 6 ++- core/benches/types/concurrent_tasks.rs | 4 +- core/benches/types/main.rs | 4 +- 9 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test_benchmark.yml diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml new file mode 100644 index 000000000000..347a18f40e24 --- /dev/null +++ b/.github/workflows/test_benchmark.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Benchmark Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/benches/**" + - "!core/src/docs/**" + - ".github/workflows/test_benchmark.yml" + # `workflow_dispatch` is needed by codspeed for first time trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + - name: Setup codspeed + run: cargo install cargo-codspeed + - name: Setup Memory env + uses: ./.github/services/memory/memory + - name: Build the benchmark targets + run: cargo codspeed build --features tests + - name: Run the benchmarks + uses: CodSpeedHQ/action@v3 + with: + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file diff --git a/core/Cargo.lock b/core/Cargo.lock index 64ba90c2a184..f0c3e7f08096 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1700,12 +1700,46 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "codspeed" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe" +dependencies = [ + "codspeed", + "colored", + "criterion", + "futures", + "tokio", +] + [[package]] name = "colorchoice" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -4794,6 +4828,7 @@ dependencies = [ "bytes", "cacache", "chrono", + "codspeed-criterion-compat", "compio", "crc32c", "criterion", diff --git a/core/Cargo.toml b/core/Cargo.toml index 50b144c5718c..395f6636a477 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -380,6 +380,7 @@ probe = { version = "0.5.1", optional = true } getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] +codspeed-criterion-compat = { version = "2.6.0", features = ["async", "async_tokio"] } criterion = { version = "0.5", features = ["async", "async_tokio"] } dotenvy = "0.15" fastrace = { version = "0.6", features = ["enable"] } diff --git a/core/benches/ops/main.rs b/core/benches/ops/main.rs index f79ce9515bee..7d0b9ba5ccc3 100644 --- a/core/benches/ops/main.rs +++ b/core/benches/ops/main.rs @@ -19,8 +19,8 @@ mod read; mod utils; mod write; -use criterion::criterion_group; -use criterion::criterion_main; +use codspeed_criterion_compat::criterion_group; +use codspeed_criterion_compat::criterion_main; criterion_group!(benches, read::bench, write::bench); criterion_main!(benches); diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs index 95204718351c..4089202fd954 100644 --- a/core/benches/ops/read.rs +++ b/core/benches/ops/read.rs @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; +use codspeed_criterion_compat::Throughput; use futures::io; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; @@ -47,7 +48,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) { let path = uuid::Uuid::new_v4().to_string(); let temp_data = TempData::generate(op.clone(), &path, content.clone()); - group.throughput(criterion::Throughput::Bytes(size.bytes() as u64)); + group.throughput(Throughput::Bytes(size.bytes() as u64)); group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| { b.to_async(&*TEST_RUNTIME).iter(|| async { let r = op.reader_with(path).await.unwrap(); @@ -84,7 +85,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) { let temp_data = TempData::generate(op.clone(), &path, content.clone()); for parallel in [1, 2, 4, 8, 16] { - group.throughput(criterion::Throughput::Bytes(parallel * size.bytes() as u64)); + group.throughput(Throughput::Bytes(parallel * size.bytes() as u64)); group.bench_with_input( format!("{}x{}", parallel, size.to_string()), &(op.clone(), &path, buf_size), diff --git a/core/benches/ops/write.rs b/core/benches/ops/write.rs index 3715ac7dc29c..34e29bf050d5 100644 --- a/core/benches/ops/write.rs +++ b/core/benches/ops/write.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; use opendal::Operator; diff --git a/core/benches/types/buffer.rs b/core/benches/types/buffer.rs index 8d9df599d6d2..23a82bc9025b 100644 --- a/core/benches/types/buffer.rs +++ b/core/benches/types/buffer.rs @@ -16,7 +16,9 @@ // under the License. use bytes::Buf; -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; +use codspeed_criterion_compat::measurement::WallTime; +use codspeed_criterion_compat::BenchmarkGroup; use opendal::Buffer; use rand::thread_rng; use size::Size; @@ -83,7 +85,7 @@ pub fn bench_non_contiguous_buffer(c: &mut Criterion) { } pub fn bench_non_contiguous_buffer_with_extreme(c: &mut Criterion) { - let mut group: criterion::BenchmarkGroup = + let mut group: BenchmarkGroup = c.benchmark_group("bench_non_contiguous_buffer_with_extreme"); let mut rng = thread_rng(); diff --git a/core/benches/types/concurrent_tasks.rs b/core/benches/types/concurrent_tasks.rs index 3962a61f2080..2fbf648d05dd 100644 --- a/core/benches/types/concurrent_tasks.rs +++ b/core/benches/types/concurrent_tasks.rs @@ -17,8 +17,8 @@ use std::time::Duration; -use criterion::BatchSize; -use criterion::Criterion; +use codspeed_criterion_compat::BatchSize; +use codspeed_criterion_compat::Criterion; use once_cell::sync::Lazy; use opendal::raw::ConcurrentTasks; use opendal::Executor; diff --git a/core/benches/types/main.rs b/core/benches/types/main.rs index 801ad814f7db..ae9f74c06f3a 100644 --- a/core/benches/types/main.rs +++ b/core/benches/types/main.rs @@ -19,8 +19,8 @@ mod buffer; mod concurrent_tasks; mod utils; -use criterion::criterion_group; -use criterion::criterion_main; +use codspeed_criterion_compat::criterion_group; +use codspeed_criterion_compat::criterion_main; criterion_group!( benches,