test: Fuzz checkpoint/checksum related parameters (#11848)

Summary:
Pull Request resolved: #11848

Remove checkpoint/checksum related parameters from fixed values and allow them to be randomized by fuzzer so more scenarios can be covered.

Differential Revision: D67172539

velox/docs/develop/testing.rst

@@ -5,6 +5,7 @@ Testing Tools
 .. toctree::
     :maxdepth: 1
 
+    testing/cache-fuzzer
     testing/fuzzer
     testing/join-fuzzer
     testing/memory-arbitration-fuzzer

velox/docs/develop/testing/async-data-cache-fuzzer.rst → velox/docs/develop/testing/cache-fuzzer.rst

@@ -49,15 +49,4 @@ Here is a full list of supported command line arguments.
 
 * ``–-ssd_cache_bytes``: Ssd cache size in bytes.
 
-* ``–-num_ssd_cache_shards``: Number of SSD cache shards.
-
-* ``–-ssd_checkpoint_interval_bytes``: Checkpoint after every
-  ``--ssd_checkpoint_interval_bytes``/``--num_ssd_cache_shards`` written into
-  each file. 0 means no checkpointing.
-
-* ``–-enable_checksum``: Enable checksum write to SSD.
-
-* ``–-enable_checksum_read_verification``: Enable checksum read verification
-  from SSD.
-
 If running from CLion IDE, add ``--logtostderr=1`` to see the full output.

velox/exec/fuzzer/CacheFuzzer.cpp

@@ -27,7 +27,7 @@
 #include "velox/common/memory/MmapAllocator.h"
 #include "velox/dwio/common/CachedBufferedInput.h"
 #include "velox/exec/tests/utils/TempDirectoryPath.h"
-#include "velox/vector/fuzzer/VectorFuzzer.h"
+#include "velox/vector/fuzzer/Utils.h"
 
 DEFINE_int32(steps, 10, "Number of plans to generate and test.");
 
@@ -60,21 +60,6 @@ DEFINE_int64(memory_cache_bytes, 32 << 20, "Memory cache size in bytes.");
 
 DEFINE_uint64(ssd_cache_bytes, 128 << 20, "Ssd cache size in bytes.");
 
-DEFINE_int32(num_ssd_cache_shards, 4, "Number of SSD cache shards.");
-
-DEFINE_uint64(
-    ssd_checkpoint_interval_bytes,
-    64 << 20,
-    "Checkpoint after every 'ssd_checkpoint_interval_bytes'/'num_ssd_cache_shards', "
-    "written into each file. 0 means no checkpointing.");
-
-DEFINE_bool(enable_checksum, true, "Enable checksum write to SSD.");
-
-DEFINE_bool(
-    enable_checksum_read_verification,
-    true,
-    "Enable checksum read verification from SSD.");
-
 using namespace facebook::velox::cache;
 using namespace facebook::velox::dwio::common;
 
@@ -188,15 +173,29 @@ void CacheFuzzer::initializeCache() {
 
   std::unique_ptr<SsdCache> ssdCache;
   if (FLAGS_ssd_cache_bytes > 0) {
+    // Enable checkpoint 75% of the time as checksum depends on it. When
+    // checkpoint is enabled, make the interval bytes larger than memory cache
+    // to prevent checkpointing too often.
+    const auto checkpointIntervalBytes = folly::Random::oneIn(4, rng_)
+        ? 0
+        : boost::random::uniform_int_distribution<uint64_t>(
+              FLAGS_memory_cache_bytes, FLAGS_max_source_file_bytes)(rng_);
+    const auto enableChecksum = folly::Random::oneIn(2, rng_);
+    const auto enableChecksumReadVerification = folly::Random::oneIn(2, rng_);
+    // Use 1-4 shards to test different cases. The number of shards shouldn't be
+    // too larger so that each shard has enough space to hold larger cache
+    // entries.
+    const auto numSsdCacheShards =
+        boost::random::uniform_int_distribution<int32_t>(1, 4)(rng_);
     SsdCache::Config config(
         fmt::format("{}/cache", sourceDataDir_->getPath()),
         FLAGS_ssd_cache_bytes,
-        FLAGS_num_ssd_cache_shards,
+        numSsdCacheShards,
         executor_.get(),
-        FLAGS_ssd_checkpoint_interval_bytes,
+        checkpointIntervalBytes,
         false,
-        FLAGS_enable_checksum,
-        FLAGS_enable_checksum_read_verification);
+        enableChecksum,
+        enableChecksumReadVerification);
     ssdCache = std::make_unique<SsdCache>(config);
   }