[KYUUBI #6041] RESTful API supports isolated authentication configuration

docs/configuration/settings.md

@@ -246,6 +246,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 | kyuubi.frontend.mysql.worker.keepalive.time            | PT1M               | Time(ms) that an idle async thread of the command execution thread pool will wait for a new task to arrive before terminating in MySQL frontend service                                                                                                                                                                                                                                                                                                                                                                     | duration | 1.4.0 |
 | kyuubi.frontend.protocols                              | THRIFT_BINARY,REST | A comma-separated list for all frontend protocols <ul> <li>THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.</li> <li>THRIFT_HTTP - HiveServer2 compatible thrift http protocol.</li> <li>REST - Kyuubi defined REST API(experimental).</li>  <li>MYSQL - MySQL compatible text protocol(experimental).</li>  <li>TRINO - Trino compatible http protocol(experimental).</li> </ul>                                                                                                                             | seq      | 1.4.0 |
 | kyuubi.frontend.proxy.http.client.ip.header            | X-Real-IP          | The HTTP header to record the real client IP address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any IP address, so it will not be used for authentication. | string   | 1.6.0 |
+| kyuubi.frontend.rest.authentication                    | NONE               | A comma-separated list of client authentication types. It fallback to `kyuubi.authentication` if not configure.                                                                                                                                                                                                                                                                                                                                                                                                             | seq      | 1.9.0 |
 | kyuubi.frontend.rest.bind.host                         | &lt;undefined&gt;  | Hostname or IP of the machine on which to run the REST frontend service.                                                                                                                                                                                                                                                                                                                                                                                                                                                    | string   | 1.4.0 |
 | kyuubi.frontend.rest.bind.port                         | 10099              | Port of the machine on which to run the REST frontend service.                                                                                                                                                                                                                                                                                                                                                                                                                                                              | int      | 1.4.0 |
 | kyuubi.frontend.rest.max.worker.threads                | 999                | Maximum number of threads in the frontend worker thread pool for the rest frontend service                                                                                                                                                                                                                                                                                                                                                                                                                                  | int      | 1.6.2 |

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -797,6 +797,14 @@ object KyuubiConf {
     .checkValues(AuthTypes)
     .createWithDefault(Seq(AuthTypes.NONE.toString))
 
+  val FRONTEND_REST_AUTHENTICATION_METHOD: ConfigEntry[Seq[String]] =
+    buildConf("kyuubi.frontend.rest.authentication")
+      .doc("A comma-separated list of client authentication types." +
+        " It fallback to `kyuubi.authentication` if not configure.")
+      .version("1.9.0")
+      .serverOnly
+      .fallbackConf(AUTHENTICATION_METHOD)
+
   val AUTHENTICATION_CUSTOM_CLASS: OptionalConfigEntry[String] =
     buildConf("kyuubi.authentication.custom.class")
       .doc("User-defined authentication implementation of " +

kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiRestFrontendService.scala

@@ -72,7 +72,7 @@ class KyuubiRestFrontendService(override val serverable: Serverable)
   private lazy val port: Int = conf.get(FRONTEND_REST_BIND_PORT)
 
   private[kyuubi] lazy val securityEnabled = {
-    val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName)
+    val authTypes = conf.get(FRONTEND_REST_AUTHENTICATION_METHOD).map(AuthTypes.withName)
     AuthUtils.kerberosEnabled(authTypes) ||
     !AuthUtils.effectivePlainAuthType(authTypes).contains(AuthTypes.NONE)
   }
@@ -103,7 +103,9 @@ class KyuubiRestFrontendService(override val serverable: Serverable)
 
   private def startInternal(): Unit = {
     val contextHandler = ApiRootResource.getServletHandler(this)
-    val holder = new FilterHolder(new AuthenticationFilter(conf))
+    val holder = new FilterHolder(new AuthenticationFilter(
+      conf,
+      conf.get(FRONTEND_REST_AUTHENTICATION_METHOD).map(AuthTypes.withName)))
     contextHandler.addFilter(holder, "/v1/*", EnumSet.allOf(classOf[DispatcherType]))
     val authenticationFactory = new KyuubiHttpAuthenticationFactory(conf)
     server.addHandler(authenticationFactory.httpHandlerWrapperFactory.wrapHandler(contextHandler))

kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala

@@ -30,11 +30,11 @@ import org.apache.hadoop.hive.shims.Utils
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
+import org.apache.kyuubi.config.KyuubiConf.{AUTHENTICATION_METHOD, FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER}
 import org.apache.kyuubi.server.http.authentication.AuthenticationFilter
 import org.apache.kyuubi.server.http.util.{CookieSigner, HttpAuthUtils, SessionManager}
 import org.apache.kyuubi.server.http.util.HttpAuthUtils.AUTHORIZATION_HEADER
-import org.apache.kyuubi.service.authentication.KyuubiAuthenticationFactory
+import org.apache.kyuubi.service.authentication.{AuthTypes, KyuubiAuthenticationFactory}
 import org.apache.kyuubi.shaded.thrift.TProcessor
 import org.apache.kyuubi.shaded.thrift.protocol.TProtocolFactory
 import org.apache.kyuubi.shaded.thrift.server.TServlet
@@ -56,7 +56,8 @@ class ThriftHttpServlet(
   private var isCookieSecure = false
   private var isHttpOnlyCookie = false
   private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For"
-  private val authenticationFilter = new AuthenticationFilter(conf)
+  private val authenticationFilter =
+    new AuthenticationFilter(conf, conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName))
 
   override def init(): Unit = {
     isCookieAuthEnabled = conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED)

kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilter.scala

@@ -26,12 +26,13 @@ import scala.collection.mutable
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{AUTHENTICATION_METHOD, FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER}
+import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
 import org.apache.kyuubi.server.http.util.HttpAuthUtils.AUTHORIZATION_HEADER
 import org.apache.kyuubi.service.authentication.{AuthTypes, InternalSecurityAccessor}
 import org.apache.kyuubi.service.authentication.AuthTypes.{KERBEROS, NOSASL}
 
-class AuthenticationFilter(conf: KyuubiConf) extends Filter with Logging {
+class AuthenticationFilter(conf: KyuubiConf, authTypes: Seq[AuthTypes.Value]) extends Filter
+  with Logging {
   import AuthenticationFilter._
   import AuthSchemes._
 
@@ -55,7 +56,6 @@ class AuthenticationFilter(conf: KyuubiConf) extends Filter with Logging {
   }
 
   private[kyuubi] def initAuthHandlers(): Unit = {
-    val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName)
     val spnegoKerberosEnabled = authTypes.contains(KERBEROS)
     val basicAuthTypeOpt = {
       if (authTypes == Set(NOSASL)) {

kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/KyuubiHttpAuthenticationFactory.scala

@@ -25,14 +25,14 @@ import org.eclipse.jetty.server.{Handler, Request}
 import org.eclipse.jetty.server.handler.HandlerWrapper
 
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{AUTHENTICATION_METHOD, ENGINE_SECURITY_ENABLED}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SECURITY_ENABLED, FRONTEND_REST_AUTHENTICATION_METHOD}
 import org.apache.kyuubi.metrics.MetricsConstants.{REST_CONN_FAIL, REST_CONN_OPEN, REST_CONN_TOTAL}
 import org.apache.kyuubi.metrics.MetricsSystem
 import org.apache.kyuubi.service.authentication.{AuthTypes, InternalSecurityAccessor}
 import org.apache.kyuubi.service.authentication.AuthTypes.KERBEROS
 
 class KyuubiHttpAuthenticationFactory(conf: KyuubiConf) {
-  private val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName)
+  private val authTypes = conf.get(FRONTEND_REST_AUTHENTICATION_METHOD).map(AuthTypes.withName)
   private val kerberosEnabled = authTypes.contains(KERBEROS)
   private val ugi = UserGroupInformation.getCurrentUser
 

kyuubi-server/src/test/scala/org/apache/kyuubi/RestClientTestHelper.scala

@@ -48,7 +48,9 @@ trait RestClientTestHelper extends RestFrontendTestHelper with KerberizedTestHel
     UserGroupInformation.setConfiguration(config)
     assert(UserGroupInformation.isSecurityEnabled)
 
-    val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "LDAP", "CUSTOM"))
+    val conf = KyuubiConf().set(
+      KyuubiConf.FRONTEND_REST_AUTHENTICATION_METHOD,
+      Seq("KERBEROS", "LDAP", "CUSTOM"))
       .set(KyuubiConf.SERVER_KEYTAB.key, testKeytab)
       .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal)
       .set(KyuubiConf.SERVER_SPNEGO_KEYTAB, testKeytab)

kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiRestAuthenticationSuite.scala

@@ -182,3 +182,33 @@ class KyuubiRestAuthenticationSuite extends RestClientTestHelper {
     assert(HttpServletResponse.SC_UNAUTHORIZED == response.getStatus)
   }
 }
+
+class NoneKyuubiRestAuthenticationSuite extends RestClientTestHelper {
+
+  override protected val otherConfigs: Map[String, String] = {
+    Map(KyuubiConf.FRONTEND_REST_AUTHENTICATION_METHOD.key -> "NONE")
+  }
+
+  test("test disable restful api authentication") {
+    val response = webTarget.path("api/v1/sessions/count")
+      .request()
+      .get()
+
+    assert(HttpServletResponse.SC_OK == response.getStatus)
+  }
+}
+
+class KerberosKyuubiRestAuthenticationSuite extends RestClientTestHelper {
+
+  override protected val otherConfigs: Map[String, String] = {
+    Map(KyuubiConf.FRONTEND_REST_AUTHENTICATION_METHOD.key -> "KERBEROS")
+  }
+
+  test("test without authorization when rest api authentication with KERBEROS") {
+    val response = webTarget.path("api/v1/sessions/count")
+      .request()
+      .get()
+
+    assert(HttpServletResponse.SC_UNAUTHORIZED == response.getStatus)
+  }
+}

kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiRestFrontendServiceSuite.scala

@@ -26,7 +26,7 @@ import org.apache.kyuubi.service.authentication.AnonymousAuthenticationProviderI
 class KyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 
   override protected lazy val conf: KyuubiConf = KyuubiConf()
-    .set(AUTHENTICATION_METHOD, Seq("NONE"))
+    .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("NONE"))
 
   test("version") {
     val resp = v1Call("version")
@@ -61,7 +61,7 @@ class KyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 class KerberosKyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 
   override protected lazy val conf: KyuubiConf = KyuubiConf()
-    .set(AUTHENTICATION_METHOD, Seq("KERBEROS"))
+    .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("KERBEROS"))
     .set(AUTHENTICATION_CUSTOM_CLASS, classOf[AnonymousAuthenticationProviderImpl].getName)
 
   test("security enabled - KERBEROS") {
@@ -72,7 +72,7 @@ class KerberosKyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 class NoneKyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 
   override protected lazy val conf: KyuubiConf = KyuubiConf()
-    .set(AUTHENTICATION_METHOD, Seq("NONE"))
+    .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("NONE"))
     .set(AUTHENTICATION_CUSTOM_CLASS, classOf[AnonymousAuthenticationProviderImpl].getName)
 
   test("security enabled - NONE") {
@@ -83,7 +83,7 @@ class NoneKyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 class KerberosAndCustomKyuubiRestFrontendServiceSuite extends RestFrontendTestHelper {
 
   override protected lazy val conf: KyuubiConf = KyuubiConf()
-    .set(AUTHENTICATION_METHOD, Seq("KERBEROS,CUSTOM"))
+    .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("KERBEROS,CUSTOM"))
     .set(AUTHENTICATION_CUSTOM_CLASS, classOf[AnonymousAuthenticationProviderImpl].getName)
 
   test("security enabled - KERBEROS,CUSTOM") {

kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/AdminResourceSuite.scala

@@ -51,7 +51,7 @@ class AdminResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper {
   private val engineMgr = new KyuubiApplicationManager()
 
   override protected lazy val conf: KyuubiConf = KyuubiConf()
-    .set(AUTHENTICATION_METHOD, Seq("CUSTOM"))
+    .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("CUSTOM"))
     .set(AUTHENTICATION_CUSTOM_CLASS, classOf[AnonymousAuthenticationProviderImpl].getName)
     .set(SERVER_ADMINISTRATORS, Set("admin001"))
     .set(ENGINE_IDLE_TIMEOUT, Duration.ofMinutes(3).toMillis)

kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala

@@ -85,7 +85,7 @@ abstract class BatchesResourceSuiteBase extends KyuubiFunSuite
   override protected lazy val conf: KyuubiConf = {
     val testResourceDir = Paths.get(sparkBatchTestResource.get).getParent
     val kyuubiConf = KyuubiConf()
-      .set(AUTHENTICATION_METHOD, Seq("CUSTOM"))
+      .set(FRONTEND_REST_AUTHENTICATION_METHOD, Seq("CUSTOM"))
       .set(AUTHENTICATION_CUSTOM_CLASS, classOf[AnonymousAuthenticationProviderImpl].getName)
       .set(SERVER_ADMINISTRATORS, Set("admin"))
       .set(BATCH_IMPL_VERSION, batchVersion)

kyuubi-server/src/test/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilterSuite.scala

@@ -19,11 +19,14 @@ package org.apache.kyuubi.server.http.authentication
 
 import org.apache.kyuubi.KyuubiFunSuite
 import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.AUTHENTICATION_METHOD
 import org.apache.kyuubi.service.authentication.AuthTypes
 
 class AuthenticationFilterSuite extends KyuubiFunSuite {
   test("add auth handler and destroy") {
-    val filter = new AuthenticationFilter(KyuubiConf())
+    val conf = KyuubiConf()
+    val filter =
+      new AuthenticationFilter(conf, conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName))
     filter.addAuthHandler(new BasicAuthenticationHandler(null))
     assert(filter.authSchemeHandlers.isEmpty)
     filter.addAuthHandler(new BasicAuthenticationHandler(AuthTypes.LDAP))

kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala

@@ -56,7 +56,7 @@ class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit {
     val id = UUID.randomUUID().toString
     conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test")
     conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L)
-    conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
+    conf.set(KyuubiConf.FRONTEND_REST_AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
     conf.set(KyuubiConf.GROUP_PROVIDER, "hadoop")
 
     val user = ldapUser

kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala

@@ -50,7 +50,7 @@ class AdminRestApiSuite extends RestClientTestHelper {
     val id = UUID.randomUUID().toString
     conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test")
     conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L)
-    conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
+    conf.set(KyuubiConf.FRONTEND_REST_AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
     conf.set(KyuubiConf.GROUP_PROVIDER, "hadoop")
     val user = ldapUser
     val engine =