HIVE-28782: Ensure Hive constructs URI's with IPv6 literals safely. (#5758) (Dmitriy Fingerman, reviewed by Ayush Saxena)

* HIVE-28782: Ensure Hive constructs URI's with IPv6 literals safely.

Author: difin

beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.hive.common.ServerUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -119,8 +120,7 @@ private void addHosts(Properties props) throws BeelineHS2ConnectionFileParseExce
     }
   }
 
-  private void addZKServiceDiscoveryHosts(Properties props)
-      throws BeelineHS2ConnectionFileParseException {
+  private void addZKServiceDiscoveryHosts(Properties props) {
     props.setProperty("serviceDiscoveryMode", "zooKeeper");
     props.setProperty("zooKeeperNamespace",
         HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE));
@@ -141,7 +141,9 @@ private void addDefaultHS2Hosts(Properties props) throws BeelineHS2ConnectionFil
     }
     int portNum = getPortNum(
         "http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE)));
-    props.setProperty("hosts", serverIPAddress.getHostName() + ":" + portNum);
+    // The hosts property is used in the constructing connection URL, serverIPAddress.getHostName() might return an 
+    // IP address depending on the configuration, hence need to properly escape a possible IPv6 literal
+    props.setProperty("hosts", IPStackUtils.concatHostPort(serverIPAddress.getHostName(), portNum));
   }
 
   private int getPortNum(boolean isHttp) {

beeline/src/test/org/apache/hive/beeline/ProxyAuthTest.java

@@ -26,10 +26,10 @@
 import java.sql.Statement;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.apache.hive.jdbc.HiveConnection;
 import org.apache.hive.service.auth.HiveAuthConstants;
 import org.apache.hive.service.cli.session.SessionUtils;
-import org.apache.hive.beeline.BeeLine;
 import org.apache.hadoop.hive.shims.Utils;
 
 /**
@@ -84,7 +84,7 @@ public static void main(String[] args) throws Exception {
     /*
      * Connect via kerberos and get delegation token
      */
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal;
+    url = String.format("jdbc:hive2://%s/default;principal=%s", IPStackUtils.concatHostPort(host, port), serverPrincipal);
     con = DriverManager.getConnection(url);
     System.out.println("Connected successfully to " + url);
     // get delegation token for the given proxy user
@@ -98,31 +98,31 @@ public static void main(String[] args) throws Exception {
     System.setProperty(BEELINE_EXIT, "true");
 
     // connect using principal via Beeline with inputStream
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal;
+    url = String.format("jdbc:hive2://%s/default;principal=%s", IPStackUtils.concatHostPort(host, port), serverPrincipal);
     currentResultFile = generateSQL(null);
     beeLineArgs = new String[] { "-u", url, "-n", "foo", "-p", "bar"};
     System.out.println("Connection with kerberos, user/password via args, using input rediction");
     BeeLine.mainWithInputRedirection(beeLineArgs, inpStream);
     compareResults( currentResultFile);
 
     // connect using principal via Beeline with inputStream
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal;
+    url = String.format("jdbc:hive2://%s/default;principal=%s", IPStackUtils.concatHostPort(host, port), serverPrincipal);
     currentResultFile = generateSQL(null);
     beeLineArgs = new String[] { "-u", url, "-n", "foo", "-p", "bar", "-f" , scriptFileName};
     System.out.println("Connection with kerberos, user/password via args, using input script");
     BeeLine.main(beeLineArgs);
     compareResults( currentResultFile);
 
     // connect using principal via Beeline with inputStream
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal;
+    url = String.format("jdbc:hive2://%s/default;principal=%s", IPStackUtils.concatHostPort(host, port), serverPrincipal);
     currentResultFile = generateSQL(url+ " foo bar ");
     beeLineArgs = new String[] { "-u", url, "-f" , scriptFileName};
     System.out.println("Connection with kerberos, user/password via connect, using input script");
     BeeLine.main(beeLineArgs);
     compareResults( currentResultFile);
 
     // connect using principal via Beeline with inputStream
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal;
+    url = String.format("jdbc:hive2://%s/default;principal=%s", IPStackUtils.concatHostPort(host, port), serverPrincipal);
     currentResultFile = generateSQL(url+ " foo bar ");
     beeLineArgs = new String[] { "-u", url, "-f" , scriptFileName};
     System.out.println("Connection with kerberos, user/password via connect, using input redirect");
@@ -134,22 +134,22 @@ public static void main(String[] args) throws Exception {
      */
     System.out.println("Store token into ugi and try");
     storeTokenInJobConf(token);
-    url = "jdbc:hive2://" + host + ":" + port + "/default;auth=delegationToken";
+    url = String.format("jdbc:hive2://%s/default;auth=delegationToken", IPStackUtils.concatHostPort(host, port));
     con = DriverManager.getConnection(url);
     System.out.println("Connecting to " + url);
     runTest();
     con.close();
 
     // connect using token via Beeline with inputStream
-    url = "jdbc:hive2://" + host + ":" + port + "/default";
+    url = String.format("jdbc:hive2://%s/default", IPStackUtils.concatHostPort(host, port));
     currentResultFile = generateSQL(null);
     beeLineArgs = new String[] { "-u", url, "-n", "foo", "-p", "bar", "-a", "delegationToken" };
     System.out.println("Connection with token, user/password via args, using input redirection");
     BeeLine.mainWithInputRedirection(beeLineArgs, inpStream);
     compareResults( currentResultFile);
 
     // connect using token via Beeline using script
-    url = "jdbc:hive2://" + host + ":" + port + "/default";
+    url = String.format("jdbc:hive2://%s/default", IPStackUtils.concatHostPort(host, port));
     currentResultFile = generateSQL(null);
     beeLineArgs = new String[] { "-u", url, "-n", "foo", "-p", "bar", "-a", "delegationToken",
         "-f", scriptFileName};
@@ -158,15 +158,15 @@ public static void main(String[] args) throws Exception {
     compareResults( currentResultFile);
 
     // connect using token via Beeline using script
-    url = "jdbc:hive2://" + host + ":" + port + "/default";
+    url = String.format("jdbc:hive2://%s/default", IPStackUtils.concatHostPort(host, port));
     currentResultFile = generateSQL(url + " foo bar ");
     beeLineArgs = new String [] {"-a", "delegationToken", "-f", scriptFileName};
     System.out.println("Connection with token, user/password via connect, using input script");
     BeeLine.main(beeLineArgs);
     compareResults( currentResultFile);
 
     // connect using token via Beeline using script
-    url = "jdbc:hive2://" + host + ":" + port + "/default";
+    url = String.format("jdbc:hive2://%s/default", IPStackUtils.concatHostPort(host, port));
     currentResultFile = generateSQL(url + " foo bar ");
     System.out.println("Connection with token, user/password via connect, using input script");
     beeLineArgs = new String [] {"-f", scriptFileName, "-a", "delegationToken"};
@@ -176,8 +176,8 @@ public static void main(String[] args) throws Exception {
     /*
      * Connect via kerberos with trusted proxy user
      */
-    url = "jdbc:hive2://" + host + ":" + port + "/default;principal=" + serverPrincipal
-          + ";hive.server2.proxy.user=" + proxyUser;
+    url = String.format("jdbc:hive2://%s/default;principal=%s;hive.server2.proxy.user=%s", 
+        IPStackUtils.concatHostPort(host, port), serverPrincipal, proxyUser);
     con = DriverManager.getConnection(url);
     System.out.println("Connected successfully to " + url);
     runTest();
@@ -191,7 +191,7 @@ public static void main(String[] args) throws Exception {
 
     /* verify the connection fails after canceling the token */
     try {
-      url = "jdbc:hive2://" + host + ":" + port + "/default;auth=delegationToken";
+      url = String.format("jdbc:hive2://%s/default;auth=delegationToken", IPStackUtils.concatHostPort(host, port));
       con = DriverManager.getConnection(url);
       throw new Exception ("connection should have failed after token cancellation");
     } catch (SQLException e) {

common/src/java/org/apache/hadoop/hive/common/auth/HiveAuthUtils.java

@@ -33,7 +33,7 @@
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Sets;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.apache.thrift.transport.TSSLTransportFactory;
 import org.apache.thrift.transport.TSSLTransportFactory.TSSLTransportParameters;
 import org.apache.thrift.transport.TServerSocket;

contrib/src/test/org/apache/hadoop/hive/contrib/serde2/TestRegexSerDe.java

@@ -28,7 +28,7 @@
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.io.Text;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import static org.junit.Assert.assertEquals;
 import org.junit.Test;
 

hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java

@@ -75,7 +75,7 @@
 import org.apache.hadoop.mapred.OutputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java

@@ -29,7 +29,7 @@
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;

hcatalog/core/src/test/java/org/apache/hive/hcatalog/MiniCluster.java

@@ -34,7 +34,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 
 /**
  * This class builds a single instance of itself with the Singleton

hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java

@@ -34,7 +34,7 @@
 
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.commons.lang3.StringUtils;
@@ -72,7 +72,6 @@
 import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
-import javax.servlet.annotation.WebFilter;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 

hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java

@@ -26,7 +26,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.shims.HadoopShimsSecure;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hive.common.IPStackUtils;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;

iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java

@@ -23,12 +23,12 @@
 import java.util.List;
 import java.util.stream.Collectors;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.IPStackUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hive.common.IPStackUtils;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.HiveSQLException;
 import org.apache.hive.service.cli.OperationHandle;