Move LZ4 codec from arrow to velox parquet writer

facebookincubator · Oct 9, 2023 · 9711ddf · 9711ddf
1 parent 36f9621
commit 9711ddf
Show file tree

Hide file tree

Showing 3 changed files with 624 additions and 12 deletions.
diff --git a/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt b/velox/dwio/parquet/writer/arrow/util/CMakeLists.txt
@@ -15,7 +15,7 @@
 add_library(
   velox_dwio_arrow_parquet_writer_util_lib
   Compression.cpp CompressionSnappy.cpp Hashing.cpp Crc32.cpp
-  CompressionZstd.cpp)
+  CompressionZstd.cpp CompressionLZ4.cpp)
 
 target_link_libraries(velox_dwio_arrow_parquet_writer_util_lib parquet arrow
-                      Snappy::snappy zstd::zstd)
+                      Snappy::snappy zstd::zstd lz4::lz4)
diff --git a/velox/dwio/parquet/writer/arrow/util/Compression.cpp b/velox/dwio/parquet/writer/arrow/util/Compression.cpp
@@ -192,19 +192,13 @@ Result<std::unique_ptr<Codec>> Codec::Create(
       break;
     }
     case Compression::LZ4:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4RawCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_FRAME:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4FrameCodec(compression_level);
-#endif
       break;
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       codec = internal::MakeLz4HadoopRawCodec();
-#endif
       break;
     case Compression::ZSTD:
       codec = internal::MakeZSTDCodec(compression_level);
@@ -256,11 +250,7 @@ bool Codec::IsAvailable(Compression::type codec_type) {
     case Compression::LZ4:
     case Compression::LZ4_FRAME:
     case Compression::LZ4_HADOOP:
-#ifdef ARROW_WITH_LZ4
       return true;
-#else
-      return false;
-#endif
     case Compression::ZSTD:
       return true;
     case Compression::BZ2: