SUBMARINE-679. Spark Security Plugin Support Spark-3.0.1

### What is this PR for? Submarine Spark Security Plugin Support Spark-3.x ### What type of PR is it? [Improvement | Feature] ### What is the Jira issue? [SUBMARINE-679](https://issues.apache.org/jira/browse/SUBMARINE-679) ### Questions: * Does this need documentation? YES Author: atovk <[email protected]> Closes #463 from atovk/SUBMARINE-679 and squashes the following commits: 56742a9 [atovk] fit scala-style check 90788fa [atovk] SubmarineSqlParser class Compatible for spark2.x and spark3.x b1309f9 [atovk] clean Dependency for test case in profiles hadoop-2.9 (hadoop3.x untested) 27eb9c2 [atovk] add doc available profiles spark-3.0 ba296c6 [atovk] fit test case in spark2.x and spark3.x eee31b5 [atovk] fix ranger in spark3.x subquery Permission filter e002c3d [atovk] fit spark 3.0 Dependency version 138f729 [atovk] add Test submarine spark security with spark 3.0 276af6c [atovk] SUBMARINE-679. Spark Security Plugin Support Spark-3.0.1
apache · Dec 8, 2020 · 3041ef2 · 3041ef2
1 parent 6ec50ec
commit 3041ef2
Show file tree

Hide file tree

Showing 20 changed files with 491 additions and 53 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -333,6 +333,46 @@ matrix:
         - TEST_FLAG=$BUILD_FLAG
         - PROFILE="-Pspark-2.4 -Pranger-2.0"
         - MODULES="-pl :submarine-spark-security"
+
+    - name: Test submarine spark security with spark 3.0 and ranger 1.0
+      language: scala
+      jdk: openjdk8
+      env:
+        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
+        - TEST_FLAG=$BUILD_FLAG
+        - PROFILE="-Pspark-3.0 -Pranger-1.0"
+        - MODULES="-pl :submarine-spark-security"
+
+    - name: Test submarine spark security with spark 3.0 and ranger 1.1
+      language: scala
+      jdk: openjdk8
+      env:
+        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
+        - TEST_FLAG=$BUILD_FLAG
+        - PROFILE="-Pspark-3.0 -Pranger-1.1"
+        - MODULES="-pl :submarine-spark-security"
+
+    - name: Test submarine spark security with spark 3.0 and ranger 1.2
+      language: scala
+      jdk: openjdk8
+      env:
+        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
+        - TEST_FLAG=$BUILD_FLAG
+        - PROFILE="-Pspark-3.0 -Pranger-1.2"
+        - MODULES="-pl :submarine-spark-security"
+
+    - name: Test submarine spark security with spark 3.0 and ranger 2.0
+      language: scala
+      jdk: openjdk8
+      env:
+        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
+        - TEST_FLAG=$BUILD_FLAG
+        - PROFILE="-Pspark-3.0 -Pranger-2.0"
+        - MODULES="-pl :submarine-spark-security"
 install:
   - mvn --version
   - echo ">>> mvn $BUILD_FLAG $MODULES $PROFILE -B"

diff --git a/docs/submarine-security/spark-security/build-submarine-spark-security-plugin.md b/docs/submarine-security/spark-security/build-submarine-spark-security-plugin.md
@@ -25,6 +25,6 @@ By default, Submarine Spark Security Plugin is built against Apache Spark 2.3.x
 
 Currently, available profiles are:
 
-Spark: -Pspark-2.3, -Pspark-2.4
+Spark: -Pspark-2.3, -Pspark-2.4, -Pspark-3.0
 
 Ranger: -Pranger-1.0, -Pranger-1.1, -Pranger-1.2 -Pranger-2.0
diff --git a/submarine-security/spark-security/pom.xml b/submarine-security/spark-security/pom.xml
@@ -46,6 +46,7 @@
     <ranger.spark.package>submarine_spark_ranger_project</ranger.spark.package>
     <ranger.version>1.1.0</ranger.version>
     <ranger.major.version>1</ranger.major.version>
+    <spark.compatible.version>2</spark.compatible.version>
     <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <scalatest.version>2.2.6</scalatest.version>
@@ -60,6 +61,11 @@
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
   </properties>
   <dependencies>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
@@ -321,6 +327,19 @@
               </sources>
             </configuration>
           </execution>
+
+          <execution>
+            <id>add-spark-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>spark-${spark.compatible.version}/src/main/scala</source>
+              </sources>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
       <plugin>
@@ -551,6 +570,20 @@
       </properties>
     </profile>
 
+    <profile>
+      <id>spark-3.0</id>
+      <properties>
+        <spark.version>3.0.1</spark.version>
+        <scala.version>2.12.10</scala.version>
+        <scala.binary.version>2.12</scala.binary.version>
+        <!--<scalatest.version>3.2.0</scalatest.version>-->
+        <spark.compatible.version>3</spark.compatible.version>
+        <commons-lang3.version>3.9</commons-lang3.version>
+        <jackson-databind.version>2.10.5</jackson-databind.version>
+        <jackson-annotations.version>2.10.5</jackson-annotations.version>
+      </properties>
+    </profile>
+
     <profile>
       <id>ranger-1.0</id>
       <properties>

diff --git a/...ecurity/spark-2/src/main/scala/org/apache/submarine/spark/compatible/CompatibleFunc.scala b/...ecurity/spark-2/src/main/scala/org/apache/submarine/spark/compatible/CompatibleFunc.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.execution.command.{AnalyzeColumnCommand, SetDatabaseCommand, ShowDatabasesCommand}
+
+object CompatibleFunc {
+
+  def getPattern(child: ShowDatabasesCommand) = child.databasePattern
+
+  def getCatLogName(s: SetDatabaseCommand) = s.databaseName
+
+  def analyzeColumnName(column: AnalyzeColumnCommand) = column.columnNames
+
+  def tableIdentifier(u: UnresolvedRelation) = u.tableIdentifier
+}
diff --git a/...ity/spark-2/src/main/scala/org/apache/submarine/spark/compatible/SubqueryCompatible.scala b/...ity/spark-2/src/main/scala/org/apache/submarine/spark/compatible/SubqueryCompatible.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery}
+
+case class SubqueryCompatible(child: LogicalPlan, correlated: Boolean= false) {
+  Subquery(child)
+}
+
+object SubqueryCompatible {
+  // def apply(child: LogicalPlan, correlated: Boolean= false) = Subquery(child)
+  def unapply(subquery: Subquery): Option[LogicalPlan] = Subquery.unapply(subquery)
+}
+
+
+
diff --git a/...rk-2/src/main/scala/org/apache/submarine/spark/compatible/command/CompatibleCommand.scala b/...rk-2/src/main/scala/org/apache/submarine/spark/compatible/command/CompatibleCommand.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.execution.command.{PersistedView, SetDatabaseCommand, ShowDatabasesCommand}
+
+package object CompatibleCommand {
+
+  type ShowDatabasesCommandCompatible = ShowDatabasesCommand
+  type SetDatabaseCommandCompatible = SetDatabaseCommand
+
+}
+
+object PersistedViewCompatible {
+  val obj: PersistedView.type = PersistedView
+}
diff --git a/.../security/parser/SubmarineSqlParser.scala → .../security/parser/SubmarineSqlParser.scala b/.../security/parser/SubmarineSqlParser.scala → .../security/parser/SubmarineSqlParser.scala
@@ -43,10 +43,10 @@ class SubmarineSqlParser(val delegate: ParserInterface) extends ParserInterface
 
   // scalastyle:off line.size.limit
   /**
-   * Fork from `org.apache.spark.sql.catalyst.parser.AbstractSqlParser#parse(java.lang.String, scala.Function1)`.
-   *
-   * @see https://github.com/apache/spark/blob/v2.4.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala#L81
-   */
+    * Fork from `org.apache.spark.sql.catalyst.parser.AbstractSqlParser#parse(java.lang.String, scala.Function1)`.
+    *
+    * @see https://github.com/apache/spark/blob/v2.4.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala#L81
+    */
   // scalastyle:on
   private def parse[T](command: String)(toResult: SubmarineSqlBaseParser => T): T = {
     val lexer = new SubmarineSqlBaseLexer(new UpperCaseCharStream(CharStreams.fromString(command)))

diff --git a/...ecurity/spark-3/src/main/scala/org/apache/submarine/spark/compatible/CompatibleFunc.scala b/...ecurity/spark-3/src/main/scala/org/apache/submarine/spark/compatible/CompatibleFunc.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.plans.logical.{SetCatalogAndNamespace, ShowNamespaces}
+import org.apache.spark.sql.execution.command.AnalyzeColumnCommand
+
+object CompatibleFunc {
+
+  def getPattern(child: ShowNamespaces) = child.pattern
+
+  def getCatLogName(s: SetCatalogAndNamespace) = s.catalogName
+
+  def analyzeColumnName(column: AnalyzeColumnCommand) = column.columnNames.get
+
+  def tableIdentifier(u: UnresolvedRelation) = TableIdentifier.apply(u.tableName)
+}
diff --git a/...ity/spark-3/src/main/scala/org/apache/submarine/spark/compatible/SubqueryCompatible.scala b/...ity/spark-3/src/main/scala/org/apache/submarine/spark/compatible/SubqueryCompatible.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery}
+
+object SubqueryCompatible {
+  def apply(child: LogicalPlan, correlated: Boolean) = Subquery(child, correlated)
+  def unapply(subquery: Subquery): Option[(LogicalPlan, Boolean)] = Subquery.unapply(subquery)
+}
+
+
diff --git a/...rk-3/src/main/scala/org/apache/submarine/spark/compatible/command/CompatibleCommand.scala b/...rk-3/src/main/scala/org/apache/submarine/spark/compatible/command/CompatibleCommand.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.spark.compatible
+
+import org.apache.spark.sql.catalyst.analysis.PersistedView
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SetCatalogAndNamespace, ShowNamespaces, Subquery}
+
+
+package object CompatibleCommand {
+
+  type ShowDatabasesCommandCompatible = ShowNamespaces
+  type SetDatabaseCommandCompatible = SetCatalogAndNamespace
+}
+
+object PersistedViewCompatible {
+  val obj: PersistedView.type = PersistedView
+}
+
+