From 67e25ac0e59b143dab6bf147b353a7dd6c022ada Mon Sep 17 00:00:00 2001 From: Zac Wen Date: Thu, 12 Dec 2024 17:39:39 -0800 Subject: [PATCH] test: Fuzz checkpoint/checksum related parameters (#11848) Summary: Pull Request resolved: https://github.com/facebookincubator/velox/pull/11848 Remove checkpoint/checksum related parameters from fixed values and allow them to be randomized by fuzzer so more scenarios can be covered. Differential Revision: D67172539 --- velox/docs/develop/testing.rst | 1 + ...data-cache-fuzzer.rst => cache-fuzzer.rst} | 11 ------ velox/exec/fuzzer/CacheFuzzer.cpp | 39 +++++++++---------- 3 files changed, 20 insertions(+), 31 deletions(-) rename velox/docs/develop/testing/{async-data-cache-fuzzer.rst => cache-fuzzer.rst} (82%) diff --git a/velox/docs/develop/testing.rst b/velox/docs/develop/testing.rst index ce18009466880..78aa8181ba45e 100644 --- a/velox/docs/develop/testing.rst +++ b/velox/docs/develop/testing.rst @@ -5,6 +5,7 @@ Testing Tools .. toctree:: :maxdepth: 1 + testing/cache-fuzzer testing/fuzzer testing/join-fuzzer testing/memory-arbitration-fuzzer diff --git a/velox/docs/develop/testing/async-data-cache-fuzzer.rst b/velox/docs/develop/testing/cache-fuzzer.rst similarity index 82% rename from velox/docs/develop/testing/async-data-cache-fuzzer.rst rename to velox/docs/develop/testing/cache-fuzzer.rst index 1259064724d57..187711243619c 100644 --- a/velox/docs/develop/testing/async-data-cache-fuzzer.rst +++ b/velox/docs/develop/testing/cache-fuzzer.rst @@ -49,15 +49,4 @@ Here is a full list of supported command line arguments. * ``–-ssd_cache_bytes``: Ssd cache size in bytes. -* ``–-num_ssd_cache_shards``: Number of SSD cache shards. - -* ``–-ssd_checkpoint_interval_bytes``: Checkpoint after every - ``--ssd_checkpoint_interval_bytes``/``--num_ssd_cache_shards`` written into - each file. 0 means no checkpointing. - -* ``–-enable_checksum``: Enable checksum write to SSD. - -* ``–-enable_checksum_read_verification``: Enable checksum read verification - from SSD. - If running from CLion IDE, add ``--logtostderr=1`` to see the full output. diff --git a/velox/exec/fuzzer/CacheFuzzer.cpp b/velox/exec/fuzzer/CacheFuzzer.cpp index 20ec5b832de08..85677431e6365 100644 --- a/velox/exec/fuzzer/CacheFuzzer.cpp +++ b/velox/exec/fuzzer/CacheFuzzer.cpp @@ -27,7 +27,7 @@ #include "velox/common/memory/MmapAllocator.h" #include "velox/dwio/common/CachedBufferedInput.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" -#include "velox/vector/fuzzer/VectorFuzzer.h" +#include "velox/vector/fuzzer/Utils.h" DEFINE_int32(steps, 10, "Number of plans to generate and test."); @@ -60,21 +60,6 @@ DEFINE_int64(memory_cache_bytes, 32 << 20, "Memory cache size in bytes."); DEFINE_uint64(ssd_cache_bytes, 128 << 20, "Ssd cache size in bytes."); -DEFINE_int32(num_ssd_cache_shards, 4, "Number of SSD cache shards."); - -DEFINE_uint64( - ssd_checkpoint_interval_bytes, - 64 << 20, - "Checkpoint after every 'ssd_checkpoint_interval_bytes'/'num_ssd_cache_shards', " - "written into each file. 0 means no checkpointing."); - -DEFINE_bool(enable_checksum, true, "Enable checksum write to SSD."); - -DEFINE_bool( - enable_checksum_read_verification, - true, - "Enable checksum read verification from SSD."); - using namespace facebook::velox::cache; using namespace facebook::velox::dwio::common; @@ -188,15 +173,29 @@ void CacheFuzzer::initializeCache() { std::unique_ptr ssdCache; if (FLAGS_ssd_cache_bytes > 0) { + // Enable checkpoint 75% of the time as checksum depends on it. When + // checkpoint is enabled, make the interval bytes larger than memory cache + // to prevent checkpointing too often. + const auto checkpointIntervalBytes = folly::Random::oneIn(4, rng_) + ? 0 + : boost::random::uniform_int_distribution( + FLAGS_memory_cache_bytes, FLAGS_max_source_file_bytes)(rng_); + const auto enableChecksum = folly::Random::oneIn(2, rng_); + const auto enableChecksumReadVerification = folly::Random::oneIn(2, rng_); + // Use 1-4 shards to test different cases. The number of shards shouldn't be + // too larger so that each shard has enough space to hold larger cache + // entries. + const auto numSsdCacheShards = + boost::random::uniform_int_distribution(1, 4)(rng_); SsdCache::Config config( fmt::format("{}/cache", sourceDataDir_->getPath()), FLAGS_ssd_cache_bytes, - FLAGS_num_ssd_cache_shards, + numSsdCacheShards, executor_.get(), - FLAGS_ssd_checkpoint_interval_bytes, + checkpointIntervalBytes, false, - FLAGS_enable_checksum, - FLAGS_enable_checksum_read_verification); + enableChecksum, + enableChecksumReadVerification); ssdCache = std::make_unique(config); }