Move LZ4 codec from arrow to velox parquet writer (6909)

facebookincubator · Oct 17, 2023 · 03dacf7 · 03dacf7
1 parent 1f809ee
commit 03dacf7
Show file tree

Hide file tree

Showing 6 changed files with 637 additions and 20 deletions.
diff --git a/velox/dwio/parquet/writer/arrow/tests/ColumnWriterTest.cpp b/velox/dwio/parquet/writer/arrow/tests/ColumnWriterTest.cpp
@@ -613,7 +613,6 @@ TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndGzipCompression) {
       Encoding::PLAIN, Compression::GZIP, false, true, LARGE_SIZE);
 }
 
-#ifdef ARROW_WITH_LZ4
 TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithLz4Compression) {
   this->TestRequiredWithSettings(
       Encoding::PLAIN, Compression::LZ4, false, false, LARGE_SIZE);
@@ -623,7 +622,6 @@ TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndLz4Compression) {
   this->TestRequiredWithSettings(
       Encoding::PLAIN, Compression::LZ4, false, true, LARGE_SIZE);
 }
-#endif
 
 TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithZstdCompression) {
   this->TestRequiredWithSettings(

diff --git a/velox/dwio/parquet/writer/arrow/tests/FileDeserializeTest.cpp b/velox/dwio/parquet/writer/arrow/tests/FileDeserializeTest.cpp
@@ -95,10 +95,8 @@ static std::vector<Compression::type> GetSupportedCodecTypes() {
 
   codec_types.push_back(Compression::GZIP);
 
-#ifdef ARROW_WITH_LZ4
   codec_types.push_back(Compression::LZ4);
   codec_types.push_back(Compression::LZ4_HADOOP);
-#endif
 
   codec_types.push_back(Compression::ZSTD);
   return codec_types;

diff --git a/velox/dwio/parquet/writer/arrow/tests/FileSerializeTest.cpp b/velox/dwio/parquet/writer/arrow/tests/FileSerializeTest.cpp
@@ -381,15 +381,13 @@ TYPED_TEST(TestSerialize, SmallFileGzip) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::GZIP));
 }
 
-#ifdef ARROW_WITH_LZ4
 TYPED_TEST(TestSerialize, SmallFileLz4) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4));
 }
 
 TYPED_TEST(TestSerialize, SmallFileLz4Hadoop) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4_HADOOP));
 }
-#endif
 
 TYPED_TEST(TestSerialize, SmallFileZstd) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::ZSTD));

diff --git a/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt b/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt
@@ -14,8 +14,19 @@
 
 add_library(
   velox_dwio_arrow_parquet_writer_util_lib
-  Compression.cpp CompressionSnappy.cpp CompressionZstd.cpp CompressionZlib.cpp
-  Hashing.cpp Crc32.cpp)
+  Compression.cpp
+  CompressionSnappy.cpp
+  CompressionZstd.cpp
+  CompressionZlib.cpp
+  CompressionLZ4.cpp
+  Hashing.cpp
+  Crc32.cpp)
 
-target_link_libraries(velox_dwio_arrow_parquet_writer_util_lib parquet arrow
-                      Snappy::snappy zstd::zstd ZLIB::ZLIB)
+target_link_libraries(
+  velox_dwio_arrow_parquet_writer_util_lib
+  parquet
+  arrow
+  Snappy::snappy
+  zstd::zstd
+  ZLIB::ZLIB
+  lz4::lz4)
diff --git a/velox/dwio/parquet/writer/arrow/util/Compression.cpp b/velox/dwio/parquet/writer/arrow/util/Compression.cpp
@@ -190,19 +190,13 @@ Result<std::unique_ptr<Codec>> Codec::Create(
       break;
     }
     case Compression::LZ4:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4RawCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_FRAME:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4FrameCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4HadoopRawCodec();
-#endif
       break;
     case Compression::ZSTD:
       codec = internal::MakeZSTDCodec(compression_level);
@@ -248,11 +242,7 @@ bool Codec::IsAvailable(Compression::type codec_type) {
     case Compression::LZ4:
     case Compression::LZ4_FRAME:
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       return true;
-#else
-      return false;
-#endif
     case Compression::ZSTD:
       return true;
     case Compression::BZ2: