Buckify various external filesystems used in HiveConnector (facebooki…

…ncubator#10479) Summary: Pull Request resolved: facebookincubator#10479 This change allows us to unblock : prestodb/presto#23082 . We can now add various external Register*Filesystem as buck dependencies which are gated by their respective flags and only enabled in OSS. Reviewed By: amitkdutta Differential Revision: D59814148 fbshipit-source-id: eb7d9478e446db5184b97f1be80dc402713b8f28
bikramSingh91 · Jul 23, 2024 · db784f6 · db784f6
1 parent afd6753
commit db784f6
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 44 deletions.
diff --git a/velox/connectors/hive/HiveConnector.cpp b/velox/connectors/hive/HiveConnector.cpp
@@ -24,27 +24,15 @@
 #include "velox/dwio/dwrf/RegisterDwrfReader.h"
 #include "velox/dwio/dwrf/RegisterDwrfWriter.h"
 
-// Meta's buck build system needs this check.
-#ifdef VELOX_ENABLE_GCS
+#include "velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" // @manual
 #include "velox/connectors/hive/storage_adapters/gcs/RegisterGCSFileSystem.h" // @manual
-#endif
-#ifdef VELOX_ENABLE_HDFS3
 #include "velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.h" // @manual
-#endif
-#ifdef VELOX_ENABLE_S3
 #include "velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.h" // @manual
-#endif
-#ifdef VELOX_ENABLE_ABFS
-#include "velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" // @manual
-#endif
 #include "velox/dwio/dwrf/reader/DwrfReader.h"
 #include "velox/dwio/dwrf/writer/Writer.h"
 #include "velox/dwio/orc/reader/OrcReader.h"
-// Meta's buck build system needs this check.
-#ifdef VELOX_ENABLE_PARQUET
 #include "velox/dwio/parquet/RegisterParquetReader.h" // @manual
 #include "velox/dwio/parquet/RegisterParquetWriter.h" // @manual
-#endif
 #include "velox/expression/FieldReference.h"
 
 #include <boost/lexical_cast.hpp>
@@ -138,24 +126,14 @@ void HiveConnectorFactory::initialize() {
     dwrf::registerDwrfReaderFactory();
     dwrf::registerDwrfWriterFactory();
     orc::registerOrcReaderFactory();
-// Meta's buck build system needs this check.
-#ifdef VELOX_ENABLE_PARQUET
+
     parquet::registerParquetReaderFactory();
     parquet::registerParquetWriterFactory();
-#endif
-// Meta's buck build system needs this check.
-#ifdef VELOX_ENABLE_S3
+
     filesystems::registerS3FileSystem();
-#endif
-#ifdef VELOX_ENABLE_HDFS3
     filesystems::registerHdfsFileSystem();
-#endif
-#ifdef VELOX_ENABLE_GCS
     filesystems::registerGCSFileSystem();
-#endif
-#ifdef VELOX_ENABLE_ABFS
     filesystems::abfs::registerAbfsFileSystem();
-#endif
     return true;
   }();
 }

diff --git a/velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.cpp b/velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.cpp
@@ -15,8 +15,8 @@
  */
 
 #ifdef VELOX_ENABLE_ABFS
-#include "velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h"
-#include "velox/connectors/hive/storage_adapters/abfs/AbfsUtil.h"
+#include "velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h" // @manual
+#include "velox/connectors/hive/storage_adapters/abfs/AbfsUtil.h" // @manual
 #include "velox/core/Config.h"
 #endif
 

diff --git a/velox/connectors/hive/storage_adapters/gcs/RegisterGCSFileSystem.cpp b/velox/connectors/hive/storage_adapters/gcs/RegisterGCSFileSystem.cpp
@@ -15,8 +15,8 @@
  */
 
 #ifdef VELOX_ENABLE_GCS
-#include "velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h"
-#include "velox/connectors/hive/storage_adapters/gcs/GCSUtil.h"
+#include "velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h" // @manual
+#include "velox/connectors/hive/storage_adapters/gcs/GCSUtil.h" // @manual
 #include "velox/core/Config.h"
 #endif
 

diff --git a/velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.cpp b/velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.cpp
@@ -17,8 +17,8 @@
 #ifdef VELOX_ENABLE_HDFS3
 #include "folly/concurrency/ConcurrentHashMap.h"
 
-#include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h"
-#include "velox/connectors/hive/storage_adapters/hdfs/HdfsUtil.h"
+#include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h" // @manual
+#include "velox/connectors/hive/storage_adapters/hdfs/HdfsUtil.h" // @manual
 #include "velox/core/Config.h"
 #include "velox/dwio/common/FileSink.h"
 #endif

diff --git a/velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.cpp b/velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.cpp
@@ -15,13 +15,13 @@
  */
 
 #ifdef VELOX_ENABLE_S3
-#include "velox/connectors/hive/HiveConfig.h"
-#include "velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h"
-#include "velox/connectors/hive/storage_adapters/s3fs/S3Util.h"
+#include "velox/connectors/hive/HiveConfig.h" // @manual
+#include "velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h" // @manual
+#include "velox/connectors/hive/storage_adapters/s3fs/S3Util.h" // @manual
 #include "velox/dwio/common/FileSink.h"
 #endif
 
-#include "velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.h"
+#include "velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.h" // @manual
 
 namespace facebook::velox::filesystems {
 

diff --git a/velox/dwio/parquet/RegisterParquetReader.cpp b/velox/dwio/parquet/RegisterParquetReader.cpp
@@ -15,7 +15,7 @@
  */
 
 #ifdef VELOX_ENABLE_PARQUET
-#include "velox/dwio/parquet/reader/ParquetReader.h"
+#include "velox/dwio/parquet/reader/ParquetReader.h" // @manual
 #endif
 
 namespace facebook::velox::parquet {

diff --git a/velox/dwio/parquet/RegisterParquetWriter.cpp b/velox/dwio/parquet/RegisterParquetWriter.cpp
@@ -15,7 +15,7 @@
  */
 
 #ifdef VELOX_ENABLE_PARQUET
-#include "velox/dwio/parquet/writer/Writer.h"
+#include "velox/dwio/parquet/writer/Writer.h" // @manual
 #endif
 
 namespace facebook::velox::parquet {

diff --git a/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp b/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp
@@ -17,17 +17,17 @@
 #include <folly/init/Init.h>
 
 #include "velox/common/base/tests/GTestUtils.h"
-#include "velox/dwio/common/tests/utils/DataFiles.h"
-#include "velox/dwio/parquet/RegisterParquetReader.h"
-#include "velox/dwio/parquet/reader/PageReader.h"
-#include "velox/dwio/parquet/reader/ParquetReader.h"
+#include "velox/dwio/common/tests/utils/DataFiles.h" // @manual
+#include "velox/dwio/parquet/RegisterParquetReader.h" // @manual
+#include "velox/dwio/parquet/reader/PageReader.h" // @manual
+#include "velox/dwio/parquet/reader/ParquetReader.h" // @manual=//velox/connectors/hive:velox_hive_connector_parquet
 #include "velox/exec/tests/utils/AssertQueryBuilder.h"
-#include "velox/exec/tests/utils/HiveConnectorTestBase.h"
+#include "velox/exec/tests/utils/HiveConnectorTestBase.h" // @manual
 #include "velox/exec/tests/utils/PlanBuilder.h"
 #include "velox/type/tests/SubfieldFiltersBuilder.h"
 
-#include "velox/connectors/hive/HiveConfig.h"
-#include "velox/dwio/parquet/writer/Writer.h"
+#include "velox/connectors/hive/HiveConfig.h" // @manual=//velox/connectors/hive:velox_hive_connector_parquet
+#include "velox/dwio/parquet/writer/Writer.h" // @manual
 
 using namespace facebook::velox;
 using namespace facebook::velox::exec;