Core: Remove gRPC support (#248)

README.md

@@ -23,6 +23,7 @@ Notes:
    extends the range of supported versions of ClickHouse Server.
 2. Since 0.6.0, HTTP becomes the default protocol.
 3. Since 0.7.0, gRPC is deprecated and not recommended, it may be removed in the future.
+4. Since 0.8.0, gRPC is removed.
 
 ## Compatible Matrix
 
@@ -62,16 +63,6 @@ Run single test
 
 `./gradlew test --tests=ConvertDistToLocalWriteSuite`
 
-### ARM Platform
+Test against custom ClickHouse image
 
-For developers/users who use ARM platform, e.g. [Apple Silicon](https://developer.apple.com/documentation/apple-silicon)
-chips, [Kunpeng](https://www.hikunpeng.com/) chips, you may not able to run TPC-DS integrations test using gRPC in local directly,
-because [ClickHouse does not provide gRPC support in official ARM image](https://github.com/ClickHouse/ClickHouse/pull/36754).
-
-As a workaround, you can set the environment variable `CLICKHOUSE_IMAGE` to use a custom image which supports gRPC
-on ARM platform for testing.
-
-```
-export CLICKHOUSE_IMAGE=pan3793/clickhouse-server:22.5.1-alpine-arm-grpc
-./gradlew clean test
-```
+`CLICKHOUSE_IMAGE=custom-org/clickhouse-server:custom-tag ./gradlew test`

clickhouse-core/src/main/scala/xenon/clickhouse/spec/NodeSpec.scala

@@ -31,8 +31,7 @@ case class NodeSpec(
   @JsonIgnore private val _host: String,
   @JsonIgnore private val _http_port: Option[Int] = None,
   @JsonIgnore private val _tcp_port: Option[Int] = None,
-  @JsonIgnore private val _grpc_port: Option[Int] = None,
-  @JsonProperty("protocol") protocol: ClickHouseProtocol = GRPC,
+  @JsonProperty("protocol") protocol: ClickHouseProtocol = HTTP,
   @JsonProperty("username") username: String = "default",
   @JsonProperty("password") password: String = "",
   @JsonProperty("database") database: String = "default",
@@ -41,7 +40,6 @@ case class NodeSpec(
   @JsonProperty("host") def host: String = findHost(_host)
   @JsonProperty("http_port") def http_port: Option[Int] = findPort(_http_port)
   @JsonProperty("tcp_port") def tcp_port: Option[Int] = findPort(_tcp_port)
-  @JsonProperty("grpc_port") def grpc_port: Option[Int] = findPort(_grpc_port)
 
   private def findHost(source: String): String =
     if (isTesting) {
@@ -56,7 +54,6 @@ case class NodeSpec(
     } else source
 
   def port: Int = protocol match {
-    case GRPC => grpc_port.get
     case HTTP => http_port.get
     case TCP => tcp_port.get
     case unsupported => throw new IllegalArgumentException(s"Unsupported protocol: $unsupported")

clickhouse-core/src/testFixtures/conf/clickhouse-cluster/config.xml

@@ -43,30 +43,13 @@
 
     <listen_host>::</listen_host>
     <http_port>8123</http_port>
-    <grpc_port>9100</grpc_port>
     <tcp_port>9000</tcp_port>
     <interserver_http_port>9009</interserver_http_port>
 
     <keep_alive_timeout>300</keep_alive_timeout>
     <max_session_timeout>3600</max_session_timeout>
     <default_session_timeout>60</default_session_timeout>
 
-    <grpc>
-        <enable_ssl>false</enable_ssl>
-
-        <!-- Default compression algorithm (applied if client doesn't specify another algorithm). Supported algorithms: none, deflate, gzip, stream_gzip -->
-        <compression>gzip</compression>
-
-        <!-- Default compression level (applied if client doesn't specify another level). Supported levels: none, low, medium, high -->
-        <compression_level>medium</compression_level>
-
-        <!-- Send/receive message size limits in bytes. -1 means unlimited -->
-        <max_send_message_size>-1</max_send_message_size>
-        <max_receive_message_size>-1</max_receive_message_size>
-
-        <verbose_logs>false</verbose_logs>
-    </grpc>
-
     <max_connections>128</max_connections>
     <max_concurrent_queries>100</max_concurrent_queries>
 

clickhouse-core/src/testFixtures/scala/xenon/clickhouse/base/ClickHouseClusterMixIn.scala

@@ -26,7 +26,6 @@ trait ClickHouseClusterMixIn extends AnyFunSuite with ForAllTestContainer {
 
   protected val ZOOKEEPER_CLIENT_PORT = 2181
   protected val CLICKHOUSE_HTTP_PORT = 8123
-  protected val CLICKHOUSE_GRPC_PORT = 9100
   protected val CLICKHOUSE_TCP_PORT = 9000
 
   protected val CLICKHOUSE_IMAGE: String = Utils.load(
@@ -35,7 +34,6 @@ trait ClickHouseClusterMixIn extends AnyFunSuite with ForAllTestContainer {
   )
 
   protected val clickhouseVersion: ClickHouseVersion = ClickHouseVersion.of(CLICKHOUSE_IMAGE.split(":").last)
-  protected val grpcEnabled: Boolean = clickhouseVersion.isNewerOrEqualTo("21.1.2.15")
 
   test("clickhouse cluster up") {
     assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s1r1").isDefined)
@@ -53,23 +51,8 @@ trait ClickHouseClusterMixIn extends AnyFunSuite with ForAllTestContainer {
     assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s2r2").isDefined)
     assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_HTTP_PORT").isDefined)
     assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_TCP_PORT").isDefined)
-
-    if (grpcEnabled) {
-      assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s1r1_PORT_$CLICKHOUSE_GRPC_PORT").isDefined)
-      assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s1r2_PORT_$CLICKHOUSE_GRPC_PORT").isDefined)
-      assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s2r1_PORT_$CLICKHOUSE_GRPC_PORT").isDefined)
-      assert(sys.props.get(s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_GRPC_PORT").isDefined)
-    }
   }
 
-  // format: off
-  val grpcExposedServices: List[ExposedService] = if (grpcEnabled)
-        ExposedService("clickhouse-s1r2", CLICKHOUSE_GRPC_PORT) ::
-        ExposedService("clickhouse-s1r1", CLICKHOUSE_GRPC_PORT) ::
-        ExposedService("clickhouse-s2r1", CLICKHOUSE_GRPC_PORT) ::
-        ExposedService("clickhouse-s2r2", CLICKHOUSE_GRPC_PORT) :: Nil else Nil
-  // format: on
-
   override val container: DockerComposeContainer = DockerComposeContainer.Def(
     composeFiles = new File(Utils.classpathResource("clickhouse-cluster/clickhouse-s2r2-compose.yml")),
     exposedServices = ExposedService("zookeeper", ZOOKEEPER_CLIENT_PORT) ::
@@ -84,30 +67,26 @@ trait ClickHouseClusterMixIn extends AnyFunSuite with ForAllTestContainer {
       ExposedService("clickhouse-s2r1", CLICKHOUSE_TCP_PORT) ::
       // s2r2
       ExposedService("clickhouse-s2r2", CLICKHOUSE_HTTP_PORT) ::
-      ExposedService("clickhouse-s2r2", CLICKHOUSE_TCP_PORT) :: grpcExposedServices,
+      ExposedService("clickhouse-s2r2", CLICKHOUSE_TCP_PORT) :: Nil,
     env = Map("CLICKHOUSE_IMAGE" -> CLICKHOUSE_IMAGE)
   ).createContainer()
 
   // format: off
   // s1r1
   def clickhouse_s1r1_host:   String = container.getServiceHost("clickhouse-s1r1", CLICKHOUSE_HTTP_PORT)
   def clickhouse_s1r1_http_port: Int = container.getServicePort("clickhouse-s1r1", CLICKHOUSE_HTTP_PORT)
-  def clickhouse_s1r1_grpc_port: Int = container.getServicePort("clickhouse-s1r1", CLICKHOUSE_GRPC_PORT)
   def clickhouse_s1r1_tcp_port:  Int = container.getServicePort("clickhouse-s1r1", CLICKHOUSE_TCP_PORT)
   // s1r2
   def clickhouse_s1r2_host:   String = container.getServiceHost("clickhouse-s1r2", CLICKHOUSE_HTTP_PORT)
   def clickhouse_s1r2_http_port: Int = container.getServicePort("clickhouse-s1r2", CLICKHOUSE_HTTP_PORT)
-  def clickhouse_s1r2_grpc_port: Int = container.getServicePort("clickhouse-s1r2", CLICKHOUSE_GRPC_PORT)
   def clickhouse_s1r2_tcp_port:  Int = container.getServicePort("clickhouse-s1r2", CLICKHOUSE_TCP_PORT)
   // s2r1
   def clickhouse_s2r1_host:   String = container.getServiceHost("clickhouse-s2r1", CLICKHOUSE_HTTP_PORT)
   def clickhouse_s2r1_http_port: Int = container.getServicePort("clickhouse-s2r1", CLICKHOUSE_HTTP_PORT)
-  def clickhouse_s2r1_grpc_port: Int = container.getServicePort("clickhouse-s2r1", CLICKHOUSE_GRPC_PORT)
   def clickhouse_s2r1_tcp_port:  Int = container.getServicePort("clickhouse-s2r1", CLICKHOUSE_TCP_PORT)
   // s2r2
   def clickhouse_s2r2_host:   String = container.getServiceHost("clickhouse-s2r2", CLICKHOUSE_HTTP_PORT)
   def clickhouse_s2r2_http_port: Int = container.getServicePort("clickhouse-s2r2", CLICKHOUSE_HTTP_PORT)
-  def clickhouse_s2r2_grpc_port: Int = container.getServicePort("clickhouse-s2r2", CLICKHOUSE_GRPC_PORT)
   def clickhouse_s2r2_tcp_port:  Int = container.getServicePort("clickhouse-s2r2", CLICKHOUSE_TCP_PORT)
   // format: on
 
@@ -129,12 +108,5 @@ trait ClickHouseClusterMixIn extends AnyFunSuite with ForAllTestContainer {
     sys.props += ((s"${PREFIX}_HOST_clickhouse-s2r2", clickhouse_s2r2_host))
     sys.props += ((s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_HTTP_PORT", clickhouse_s2r2_http_port.toString))
     sys.props += ((s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_TCP_PORT", clickhouse_s2r2_tcp_port.toString))
-    // all grpc
-    if (grpcEnabled) {
-      sys.props += ((s"${PREFIX}_HOST_clickhouse-s1r1_PORT_$CLICKHOUSE_GRPC_PORT", clickhouse_s1r1_grpc_port.toString))
-      sys.props += ((s"${PREFIX}_HOST_clickhouse-s1r2_PORT_$CLICKHOUSE_GRPC_PORT", clickhouse_s1r2_grpc_port.toString))
-      sys.props += ((s"${PREFIX}_HOST_clickhouse-s2r1_PORT_$CLICKHOUSE_GRPC_PORT", clickhouse_s2r1_grpc_port.toString))
-      sys.props += ((s"${PREFIX}_HOST_clickhouse-s2r2_PORT_$CLICKHOUSE_GRPC_PORT", clickhouse_s2r2_grpc_port.toString))
-    }
   }
 }

clickhouse-core/src/testFixtures/scala/xenon/clickhouse/base/ClickHouseSingleMixIn.scala

@@ -31,12 +31,10 @@ trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer {
   val CLICKHOUSE_DB:       String = Utils.load("CLICKHOUSE_DB", "")
 
   private val CLICKHOUSE_HTTP_PORT = 8123
-  private val CLICKHOUSE_GRPC_PORT = 9100
   private val CLICKHOUSE_TPC_PORT  = 9000
   // format: on
 
   protected val clickhouseVersion: ClickHouseVersion = ClickHouseVersion.of(CLICKHOUSE_IMAGE.split(":").last)
-  protected val grpcEnabled: Boolean = clickhouseVersion.isNewerOrEqualTo("21.1.2.15")
 
   protected val rootProjectDir: Path = {
     val thisClassURI = this.getClass.getProtectionDomain.getCodeSource.getLocation.toURI
@@ -62,11 +60,7 @@ trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer {
         .withEnv("CLICKHOUSE_USER", CLICKHOUSE_USER)
         .withEnv("CLICKHOUSE_PASSWORD", CLICKHOUSE_PASSWORD)
         .withEnv("CLICKHOUSE_DB", CLICKHOUSE_DB)
-        .withExposedPorts(CLICKHOUSE_HTTP_PORT, CLICKHOUSE_GRPC_PORT, CLICKHOUSE_TPC_PORT)
-        .withCopyFileToContainer(
-          MountableFile.forClasspathResource("clickhouse-single/grpc_config.xml"),
-          "/etc/clickhouse-server/config.d/grpc_config.xml"
-        )
+        .withExposedPorts(CLICKHOUSE_HTTP_PORT, CLICKHOUSE_TPC_PORT)
         .withFileSystemBind(s"${sys.env("ROOT_PROJECT_DIR")}/log/clickhouse-server", "/var/log/clickhouse-server")
         .withCopyFileToContainer(
           MountableFile.forClasspathResource("clickhouse-single/users.xml"),
@@ -77,7 +71,6 @@ trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer {
   // format: off
   def clickhouseHost:  String = container.host
   def clickhouseHttpPort: Int = container.mappedPort(CLICKHOUSE_HTTP_PORT)
-  def clickhouseGrpcPort: Int = container.mappedPort(CLICKHOUSE_GRPC_PORT)
   def clickhouseTcpPort:  Int = container.mappedPort(CLICKHOUSE_TPC_PORT)
   // format: on
 }

deploy.gradle

@@ -45,7 +45,7 @@ subprojects {
                         pom {
                             name = "Spark ClickHouse Connector"
                             url = "https://github.com/housepower/spark-clickhouse-connector"
-                            description = "Spark ClickHouse Connector build on DataSourceV2 API and gRPC protocol."
+                            description = "Spark ClickHouse Connector build on Apache Spark DataSourceV2 API."
                             developers {
                                 developer {
                                     id = 'pan3793'

docs/developers/01_build_and_test.md

@@ -48,16 +48,6 @@ Run single test
 
 `./gradlew test --tests=ConvertDistToLocalWriteSuite`
 
-### ARM Platform
+Test against custom ClickHouse image
 
-For developers/users who use ARM platform, e.g. [Apple Silicon](https://developer.apple.com/documentation/apple-silicon)
-chips, [Kunpeng](https://www.hikunpeng.com/) chips, you may not able to run TPC-DS integrations test using gRPC in local directly,
-because [ClickHouse does not provide gRPC support in official ARM image](https://github.com/ClickHouse/ClickHouse/pull/36754).
-
-As a workaround, you can set the environment variable `CLICKHOUSE_IMAGE` to use a custom image which supports gRPC
-on ARM platform for testing.
-
-```
-export CLICKHOUSE_IMAGE=pan3793/clickhouse-server:22.5.1-alpine-arm-grpc
-./gradlew clean test
-```
+`CLICKHOUSE_IMAGE=custom-org/clickhouse-server:custom-tag ./gradlew test`