Move LZ4 codec from arrow to velox parquet writer

facebookincubator · Oct 9, 2023 · 8bd29e1 · 8bd29e1
1 parent 36f9621
commit 8bd29e1
Show file tree

Hide file tree

Showing 5 changed files with 624 additions and 16 deletions.
diff --git a/velox/dwio/parquet/writer/arrow/tests/FileDeserializeTest.cpp b/velox/dwio/parquet/writer/arrow/tests/FileDeserializeTest.cpp
@@ -97,10 +97,8 @@ static std::vector<Compression::type> GetSupportedCodecTypes() {
   codec_types.push_back(Compression::GZIP);
 #endif
 
-#ifdef ARROW_WITH_LZ4
   codec_types.push_back(Compression::LZ4);
   codec_types.push_back(Compression::LZ4_HADOOP);
-#endif
 
   codec_types.push_back(Compression::ZSTD);
   return codec_types;

diff --git a/velox/dwio/parquet/writer/arrow/tests/FileSerializeTest.cpp b/velox/dwio/parquet/writer/arrow/tests/FileSerializeTest.cpp
@@ -383,15 +383,13 @@ TYPED_TEST(TestSerialize, SmallFileGzip) {
 }
 #endif
 
-#ifdef ARROW_WITH_LZ4
 TYPED_TEST(TestSerialize, SmallFileLz4) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4));
 }
 
 TYPED_TEST(TestSerialize, SmallFileLz4Hadoop) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4_HADOOP));
 }
-#endif
 
 TYPED_TEST(TestSerialize, SmallFileZstd) {
   ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::ZSTD));

diff --git a/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt b/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt
@@ -15,7 +15,7 @@
 add_library(
   velox_dwio_arrow_parquet_writer_util_lib
   Compression.cpp CompressionSnappy.cpp Hashing.cpp Crc32.cpp
-  CompressionZstd.cpp)
+  CompressionZstd.cpp CompressionLZ4.cpp)
 
 target_link_libraries(velox_dwio_arrow_parquet_writer_util_lib parquet arrow
-                      Snappy::snappy zstd::zstd)
+                      Snappy::snappy zstd::zstd lz4::lz4)
diff --git a/velox/dwio/parquet/writer/arrow/util/Compression.cpp b/velox/dwio/parquet/writer/arrow/util/Compression.cpp
@@ -192,19 +192,13 @@ Result<std::unique_ptr<Codec>> Codec::Create(
       break;
     }
     case Compression::LZ4:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4RawCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_FRAME:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4FrameCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4HadoopRawCodec();
-#endif
       break;
     case Compression::ZSTD:
       codec = internal::MakeZSTDCodec(compression_level);
@@ -256,11 +250,7 @@ bool Codec::IsAvailable(Compression::type codec_type) {
     case Compression::LZ4:
     case Compression::LZ4_FRAME:
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       return true;
-#else
-      return false;
-#endif
     case Compression::ZSTD:
       return true;
     case Compression::BZ2: