From b5b42bc2e2ce5f427627378d63664f7530a48898 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 7 Jan 2024 12:42:57 +0800 Subject: [PATCH] ci: Refactor fuzz to integrate with test planner (#3936) * ci: Refactor fuzz to integrate with test planner Signed-off-by: Xuanwo * Fix Signed-off-by: Xuanwo * Handling write-min-size Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- .github/actions/fuzz_test/action.yaml | 49 +++++++ .github/workflows/fuzz_test.yml | 176 +++----------------------- Cargo.lock | 1 + core/fuzz/Cargo.toml | 57 ++++++++- core/fuzz/fuzz_reader.rs | 25 +++- core/fuzz/fuzz_reader_with_buffer.rs | 164 ------------------------ core/fuzz/fuzz_writer.rs | 10 +- 7 files changed, 153 insertions(+), 329 deletions(-) create mode 100644 .github/actions/fuzz_test/action.yaml delete mode 100644 core/fuzz/fuzz_reader_with_buffer.rs diff --git a/.github/actions/fuzz_test/action.yaml b/.github/actions/fuzz_test/action.yaml new file mode 100644 index 000000000000..bad0a89fa242 --- /dev/null +++ b/.github/actions/fuzz_test/action.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Fuzz Test +description: 'Fuzz test given setup and service' +inputs: + setup: + description: "The setup action for test" + service: + description: "The service to test" + target: + description: "The fuzz target to test" + +runs: + using: "composite" + steps: + - name: Setup + shell: bash + run: | + mkdir -p ./dynamic_fuzz_test && + cat <./dynamic_fuzz_test/action.yml + runs: + using: composite + steps: + - name: Setup Fuzz Test + uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }} + - name: Run Fuzz Test + shell: bash + working-directory: core/fuzz + run: cargo +nightly fuzz run ${{ inputs.target }} --features services-${{ inputs.service }} -- -max_total_time=120 + env: + OPENDAL_TEST: ${{ inputs.service }} + EOF + - name: Run + uses: ./dynamic_fuzz_test diff --git a/.github/workflows/fuzz_test.yml b/.github/workflows/fuzz_test.yml index 97652f040e31..a67e783fbf84 100644 --- a/.github/workflows/fuzz_test.yml +++ b/.github/workflows/fuzz_test.yml @@ -28,10 +28,6 @@ on: - "core/src/**" - "core/fuzz/**" - "!core/src/docs/**" - - "!core/src/services/**" - - "core/src/services/fs/**" - - "core/src/services/memory/**" - - "core/src/services/s3/**" - ".github/workflows/fuzz_test.yml" concurrency: @@ -39,164 +35,32 @@ concurrency: cancel-in-progress: true jobs: - fuzz-build: + fuzz: + name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }} / ${{ matrix.target }} runs-on: ubuntu-latest + strategy: + matrix: + target: [ "fuzz_reader", "fuzz_writer" ] + cases: + - { service: "memory", setup: "memory" } + - { service: "fs", setup: "local_fs" } + - { service: "s3", setup: "minio_s3" } + - { service: "azblob", setup: "azurite_azblob" } steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup - - name: Install libfuzz - shell: bash - run: sudo apt-get install -y libfuzzer-14-dev - - - name: Install cargo fuzz + - name: Set Rust Fuzz shell: bash - run: rustup install nightly && cargo +nightly install cargo-fuzz - - name: Build Fuzz Targets - shell: bash - working-directory: core/fuzz - run: cargo +nightly fuzz build + run: | + sudo apt-get install -y libfuzzer-14-dev + rustup install nightly + cargo +nightly install cargo-fuzz + - name: Fuzz + uses: ./.github/actions/fuzz_test env: CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a - - - name: Upload build artifacts - uses: actions/upload-artifact@v3 with: - name: fuzz_targets - path: | - ./target/x86_64-unknown-linux-gnu/release/fuzz_reader - ./target/x86_64-unknown-linux-gnu/release/fuzz_reader_with_buffer - ./target/x86_64-unknown-linux-gnu/release/fuzz_writer - - fuzz-test-s3: - runs-on: ubuntu-latest - needs: fuzz-build - services: - minio: - image: wktk/minio-server - ports: - - 9000:9000 - env: - MINIO_ACCESS_KEY: "minioadmin" - MINIO_SECRET_KEY: "minioadmin" - strategy: - fail-fast: true - matrix: - fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer] - steps: - - name: Install libfuzzer - run: sudo apt-get install -y libfuzzer-14-dev - - name: Setup Test Bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - name: Download Fuzz Targets - uses: actions/download-artifact@v3 - with: - name: fuzz_targets - path: ./target - - name: Grant Execute Permissions - shell: bash - run: chmod +x ./target/${{ matrix.fuzz-targets }} - - name: Run Fuzz Test - shell: bash - run: ./target/${{ matrix.fuzz-targets }} -max_total_time=120 - env: - OPENDAL_S3_TEST: on - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" - OPENDAL_S3_ACCESS_KEY_ID: minioadmin - OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin - OPENDAL_S3_REGION: us-east-1 - - fuzz-test-fs: - runs-on: ubuntu-latest - needs: fuzz-build - strategy: - fail-fast: true - matrix: - fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer] - steps: - - name: Install libfuzzer - run: sudo apt-get install -y libfuzzer-14-dev - - name: Download Fuzz Targets - uses: actions/download-artifact@v3 - with: - name: fuzz_targets - path: ./target - - name: Grant Execute Permissions - shell: bash - run: chmod +x ./target/${{ matrix.fuzz-targets }} - - name: Run Fuzz Test - shell: bash - run: ./target/${{ matrix.fuzz-targets }} -max_total_time=120 - env: - OPENDAL_FS_TEST: on - OPENDAL_FS_ROOT: ${{ runner.temp }}/ - - fuzz-test-memory: - runs-on: ubuntu-latest - needs: fuzz-build - strategy: - fail-fast: true - matrix: - fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer] - steps: - - name: Install libfuzzer - run: sudo apt-get install -y libfuzzer-14-dev - - name: Download Fuzz Targets - uses: actions/download-artifact@v3 - with: - name: fuzz_targets - path: ./target - - name: Grant Execute Permissions - shell: bash - run: chmod +x ./target/${{ matrix.fuzz-targets }} - - name: Run Fuzz Test - shell: bash - run: ./target/${{ matrix.fuzz-targets }} -max_total_time=120 - env: - OPENDAL_MEMORY_TEST: on - - - fuzz-test-azure: - runs-on: ubuntu-latest - needs: fuzz-build - services: - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10000 - strategy: - fail-fast: true - matrix: - fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer] - steps: - - name: Install libfuzzer - run: sudo apt-get install -y libfuzzer-14-dev - - name: Setup test bucket - shell: bash - run: | - az storage container create \ - --name test \ - --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - - name: Download Fuzz Targets - uses: actions/download-artifact@v3 - with: - name: fuzz_targets - path: ./target - - name: Grant Execute Permissions - shell: bash - run: chmod +x ./target/${{ matrix.fuzz-targets }} - - name: Run Fuzz Test - shell: bash - run: ./target/${{ matrix.fuzz-targets }} -max_total_time=120 - env: - OPENDAL_AZBLOB_TEST: on - OPENDAL_AZBLOB_CONTAINER: test - OPENDAL_AZBLOB_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1" - OPENDAL_AZBLOB_ACCOUNT_NAME: devstoreaccount1 - OPENDAL_AZBLOB_ACCOUNT_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - + setup: ${{ matrix.cases.setup }} + service: ${{ matrix.cases.service }} + target: ${{ matrix.target }} diff --git a/Cargo.lock b/Cargo.lock index 4912f1cea208..ba191452b03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4669,6 +4669,7 @@ dependencies = [ "libfuzzer-sys", "opendal", "tokio", + "tracing", "tracing-subscriber", "uuid", ] diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml index e6b34e879085..2e8a9fb050be 100644 --- a/core/fuzz/Cargo.toml +++ b/core/fuzz/Cargo.toml @@ -26,6 +26,58 @@ license.workspace = true [package.metadata] cargo-fuzz = true +[features] +services-alluxio = ["opendal/services-alluxio"] +services-azblob = ["opendal/services-azblob"] +services-azdls = ["opendal/services-azdls"] +services-azfile = ["opendal/services-azfile"] +services-b2 = ["opendal/services-b2"] +services-cacache = ["opendal/services-cacache"] +services-cos = ["opendal/services-cos"] +services-dashmap = ["opendal/services-dashmap"] +services-dropbox = ["opendal/services-dropbox"] +services-etcd = ["opendal/services-etcd"] +services-foundationdb = ["opendal/services-foundationdb"] +services-fs = ["opendal/services-fs"] +services-ftp = ["opendal/services-ftp"] +services-gcs = ["opendal/services-gcs"] +services-gdrive = ["opendal/services-gdrive"] +services-ghac = ["opendal/services-ghac"] +services-gridfs = ["opendal/services-gridfs"] +services-hdfs = ["opendal/services-hdfs"] +services-http = ["opendal/services-http"] +services-huggingface = ["opendal/services-huggingface"] +services-ipfs = ["opendal/services-ipfs"] +services-ipmfs = ["opendal/services-ipmfs"] +services-libsql = ["opendal/services-libsql"] +services-memcached = ["opendal/services-memcached"] +services-memory = ["opendal/services-memory"] +services-mini-moka = ["opendal/services-mini-moka"] +services-moka = ["opendal/services-moka"] +services-mongodb = ["opendal/services-mongodb"] +services-mysql = ["opendal/services-mysql"] +services-obs = ["opendal/services-obs"] +services-onedrive = ["opendal/services-onedrive"] +services-oss = ["opendal/services-oss"] +services-persy = ["opendal/services-persy"] +services-postgresql = ["opendal/services-postgresql"] +services-redb = ["opendal/services-redb"] +services-redis = ["opendal/services-redis"] +services-rocksdb = ["opendal/services-rocksdb"] +services-s3 = ["opendal/services-s3"] +services-seafile = ["opendal/services-seafile"] +services-sftp = ["opendal/services-sftp"] +services-sled = ["opendal/services-sled"] +services-sqlite = ["opendal/services-sqlite"] +services-supabase = ["opendal/services-supabase"] +services-swift = ["opendal/services-swift"] +services-tikv = ["opendal/services-tikv"] +services-upyun = ["opendal/services-upyun"] +services-vercel-artifacts = ["opendal/services-vercel-artifacts"] +services-wasabi = ["opendal/services-wasabi"] +services-webdav = ["opendal/services-webdav"] +services-webhdfs = ["opendal/services-webhdfs"] + [dependencies] anyhow = "1.0.71" arbitrary = { version = "1.3.0", features = ["derive"] } @@ -34,16 +86,13 @@ dotenvy = "0.15.6" libfuzzer-sys = "0.4" opendal = { path = "..", features = ["tests"] } tokio = { version = "1", features = ["full"] } +tracing = "0.1" tracing-subscriber = { version = "0.3", features = [ "env-filter", "tracing-log", ] } uuid = { version = "1", features = ["v4"] } -[[bin]] -name = "fuzz_reader_with_buffer" -path = "fuzz_reader_with_buffer.rs" - [[bin]] name = "fuzz_reader" path = "fuzz_reader.rs" diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs index b0f83fc03b60..2d9c0efa846a 100644 --- a/core/fuzz/fuzz_reader.rs +++ b/core/fuzz/fuzz_reader.rs @@ -31,6 +31,7 @@ use opendal::raw::tests::TEST_RUNTIME; use opendal::raw::BytesRange; use opendal::Operator; use opendal::Result; +use tracing::warn; const MAX_DATA_SIZE: usize = 16 * 1024 * 1024; @@ -39,6 +40,7 @@ struct FuzzInput { path: String, size: usize, range: BytesRange, + buffer: Option, actions: Vec, } @@ -53,6 +55,7 @@ impl Debug for FuzzInput { .field("path", &self.path) .field("size", &self.size) .field("range", &self.range.to_string()) + .field("buffer", &self.buffer) .field("actions", &actions) .finish() } @@ -83,6 +86,12 @@ impl Arbitrary<'_> for FuzzInput { }; let range = BytesRange::new(offset, size); + let buffer = if u.int_in_range(0..=1)? == 1 { + Some(u.int_in_range(1..=8 * 1024 * 1024)?) + } else { + None + }; + let count = u.int_in_range(1..=1024)?; let mut actions = vec![]; @@ -121,6 +130,7 @@ impl Arbitrary<'_> for FuzzInput { path: uuid::Uuid::new_v4().to_string(), size: total_size, range, + buffer, actions, }) } @@ -130,10 +140,12 @@ async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> { let mut checker = ReadChecker::new(input.size, input.range); op.write(&input.path, checker.data()).await?; - let r = op - .reader_with(&input.path) - .range(input.range.to_range()) - .await?; + let mut r = op.reader_with(&input.path); + r = r.range(input.range.to_range()); + if let Some(buffer) = input.buffer { + r = r.buffer(buffer); + } + let r = r.await?; checker.check(r, &input.actions).await; @@ -150,6 +162,11 @@ fuzz_target!(|input: FuzzInput| { let op = init_test_service().expect("operator init must succeed"); if let Some(op) = op { + if !op.info().full_capability().read_with_range { + warn!("service doesn't support read with range, skip fuzzing"); + return; + } + TEST_RUNTIME.block_on(async { fuzz_reader(op, input.clone()) .await diff --git a/core/fuzz/fuzz_reader_with_buffer.rs b/core/fuzz/fuzz_reader_with_buffer.rs deleted file mode 100644 index aa1baf6f8327..000000000000 --- a/core/fuzz/fuzz_reader_with_buffer.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![no_main] - -use std::fmt::Debug; -use std::fmt::Formatter; -use std::io::SeekFrom; - -use libfuzzer_sys::arbitrary::Arbitrary; -use libfuzzer_sys::arbitrary::Unstructured; -use libfuzzer_sys::fuzz_target; -use opendal::raw::tests::init_test_service; -use opendal::raw::tests::ReadAction; -use opendal::raw::tests::ReadChecker; -use opendal::raw::tests::TEST_RUNTIME; -use opendal::raw::BytesRange; -use opendal::Operator; -use opendal::Result; - -const MAX_DATA_SIZE: usize = 16 * 1024 * 1024; - -#[derive(Clone)] -struct FuzzInput { - path: String, - size: usize, - range: BytesRange, - actions: Vec, - buffer: usize, -} - -impl Debug for FuzzInput { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut actions = self.actions.clone(); - // Remove all Read(0) entry. - let empty = ReadAction::Read(0); - actions.retain(|e| e != &empty); - - f.debug_struct("FuzzInput") - .field("path", &self.path) - .field("size", &self.size) - .field("buffer", &self.buffer) - .field("range", &self.range.to_string()) - .field("actions", &actions) - .finish() - } -} - -impl Arbitrary<'_> for FuzzInput { - fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { - let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?; - let buffer = u.int_in_range(1..=MAX_DATA_SIZE + 1024)?; - - // TODO: it's valid that size is larger than total_size. - let (offset, size) = match u.int_in_range(0..=3)? { - // Full range - 0 => (None, None), - 1 => { - let offset = u.int_in_range(0..=total_size as u64 - 1)?; - (Some(offset), None) - } - 2 => { - let size = u.int_in_range(1..=total_size as u64)?; - (None, Some(size)) - } - 3 => { - let offset = u.int_in_range(0..=total_size as u64 - 1)?; - let size = u.int_in_range(1..=total_size as u64 - offset)?; - (Some(offset), Some(size)) - } - _ => unreachable!("invalid int generated by arbitrary"), - }; - let range = BytesRange::new(offset, size); - - let count = u.int_in_range(1..=1024)?; - let mut actions = vec![]; - - for _ in 0..count { - let action = match u.int_in_range(0..=4)? { - // Read - 0 => { - let size = u.int_in_range(0..=total_size * 2)?; - ReadAction::Read(size) - } - // Next - 1 => ReadAction::Next, - // Seek Start - 2 => { - // NOTE: seek out of the end of file is valid. - let offset = u.int_in_range(0..=total_size * 2)?; - ReadAction::Seek(SeekFrom::Start(offset as u64)) - } - // Seek Current - 3 => { - let offset = u.int_in_range(-(total_size as i64)..=(total_size as i64))?; - ReadAction::Seek(SeekFrom::Current(offset)) - } - // Seek End - 4 => { - let offset = u.int_in_range(-(total_size as i64)..=(total_size as i64))?; - ReadAction::Seek(SeekFrom::End(offset)) - } - _ => unreachable!("invalid int generated by arbitrary"), - }; - - actions.push(action); - } - - Ok(FuzzInput { - path: uuid::Uuid::new_v4().to_string(), - size: total_size, - range, - actions, - buffer, - }) - } -} - -async fn fuzz_reader_with_buffer(op: Operator, input: FuzzInput) -> Result<()> { - let mut checker = ReadChecker::new(input.size, input.range); - op.write(&input.path, checker.data()).await?; - - let r = op - .reader_with(&input.path) - .range(input.range.to_range()) - .buffer(input.buffer) - .await?; - - checker.check(r, &input.actions).await; - - op.delete(&input.path).await?; - Ok(()) -} - -fuzz_target!(|input: FuzzInput| { - let _ = tracing_subscriber::fmt() - .pretty() - .with_test_writer() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .try_init(); - - let op = init_test_service().expect("operator init must succeed"); - if let Some(op) = op { - TEST_RUNTIME.block_on(async { - fuzz_reader_with_buffer(op, input.clone()) - .await - .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}")); - }) - } -}); diff --git a/core/fuzz/fuzz_writer.rs b/core/fuzz/fuzz_writer.rs index df40e84590a4..02183311e203 100644 --- a/core/fuzz/fuzz_writer.rs +++ b/core/fuzz/fuzz_writer.rs @@ -26,6 +26,7 @@ use opendal::raw::tests::WriteChecker; use opendal::raw::tests::TEST_RUNTIME; use opendal::Operator; use opendal::Result; +use tracing::warn; const MAX_DATA_SIZE: usize = 16 * 1024 * 1024; @@ -50,7 +51,7 @@ impl Arbitrary<'_> for FuzzInput { None }; - let count = u.int_in_range(128..=1024)?; + let count = u.int_in_range(1..=1024)?; for _ in 0..count { let size = u.int_in_range(1..=MAX_DATA_SIZE)?; @@ -81,6 +82,8 @@ async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> { let mut writer = op.writer_with(&path); if let Some(buffer) = input.buffer { writer = writer.buffer(buffer); + } else if let Some(min_size) = op.info().full_capability().write_multi_min_size { + writer = writer.buffer(min_size); } if let Some(concurrent) = input.concurrent { writer = writer.concurrent(concurrent); @@ -111,6 +114,11 @@ fuzz_target!(|input: FuzzInput| { let op = init_test_service().expect("operator init must succeed"); if let Some(op) = op { + if !op.info().full_capability().write_can_multi { + warn!("service doesn't support write multi, skip fuzzing"); + return; + } + TEST_RUNTIME.block_on(async { fuzz_writer(op, input.clone()) .await