HIVE-5312: Let HiveServer2 run simultaneously in HTTP (over thrift) a…

…nd Binary (normal thrift transport) mode (Adam Szita, based on original patch from Narendra Penagulur, reviewed by Peter Vary)

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -3035,7 +3035,10 @@ public static enum ConfVars {
     HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
         "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" +
         "property is set, the value must be a valid path to an init file or directory where the init file is located."),
-    HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new StringSet("binary", "http"),
+    HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode",
+        HiveServer2TransportMode.binary.toString(),
+        new StringSet(HiveServer2TransportMode.binary.toString(),
+            HiveServer2TransportMode.http.toString(), HiveServer2TransportMode.all.toString()),
         "Transport mode of HiveServer2."),
     HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "",
         "Bind host on which to run the HiveServer2 Thrift service."),

common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.conf;
+
+/**
+ * Hive Transport mode.
+ */
+public enum HiveServer2TransportMode {
+  /**
+   * Three modes: http, binary or all (which includes binary as well as http).
+   */
+  http, binary, all
+}

itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java

@@ -180,13 +180,19 @@ public static MiniHS2 getMiniHS2WithKerb(MiniHiveKdc miniHiveKdc, HiveConf hiveC
   */
  public static MiniHS2 getMiniHS2WithKerb(MiniHiveKdc miniHiveKdc, HiveConf hiveConf,
      String authType) throws Exception {
-   String hivePrincipal =
-       miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL);
-   String hiveKeytab = miniHiveKdc.getKeyTabFile(
-       miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL));
-
-   return new MiniHS2.Builder().withConf(hiveConf).withMiniKdc(hivePrincipal, hiveKeytab).
-       withAuthenticationType(authType).build();
+    String hivePrincipal =
+            miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL);
+    String hiveKeytab = miniHiveKdc.getKeyTabFile(
+            miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL));
+
+    MiniHS2.Builder miniHS2Builder = new MiniHS2.Builder()
+                                              .withConf(hiveConf)
+                                              .withMiniKdc(hivePrincipal, hiveKeytab)
+                                              .withAuthenticationType(authType);
+    if (HiveServer2.isHttpTransportMode(hiveConf)) {
+      miniHS2Builder.withHTTPTransport();
+    }
+    return miniHS2Builder.build();
  }
 
   /**

itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java

@@ -21,7 +21,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
@@ -30,56 +33,68 @@
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hive.jdbc.HiveConnection;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
+
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+/**
+ * Testing JDBC with Mini KDC.
+ */
+@RunWith(Parameterized.class)
 public class TestJdbcWithMiniKdcCookie {
   private static MiniHS2 miniHS2 = null;
-  private static MiniHiveKdc miniHiveKdc = null;
-  private Connection hs2Conn;
+  private MiniHiveKdc miniHiveKdc = null;
+  private static Connection hs2Conn;
   File dataFile;
   protected static HiveConf hiveConf;
   private static String HIVE_NON_EXISTENT_USER = "hive_no_exist";
 
+  @Parameterized.Parameter
+  public String transportMode =  null;
+
+  @Parameterized.Parameters(name = "{index}: tranportMode={0}")
+  public static Collection<Object[]> transportModes() {
+    return Arrays.asList(new Object[][]{{MiniHS2.HS2_ALL_MODE}, {MiniHS2.HS2_HTTP_MODE}});
+  }
+
   @BeforeClass
   public static void beforeTest() throws Exception {
+    Class.forName(MiniHS2.getJdbcDriverName());
+  }
+
+  @Before
+  public void setUp() throws Exception {
     miniHiveKdc = new MiniHiveKdc();
+    DriverManager.setLoginTimeout(0);
     hiveConf = new HiveConf();
-    hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, MiniHS2.HS2_HTTP_MODE);
+    hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode);
     System.err.println("Testing using HS2 mode : "
-      + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE));
+        + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE));
     hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED,
-      true);
+        true);
     // set a small time unit as cookie max age so that the server sends a 401
     hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE,
-      1, TimeUnit.SECONDS);
+        1, TimeUnit.SECONDS);
     hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
     miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf);
     miniHS2.start(new HashMap<String, String>());
   }
 
-  @Before
-  public void setUp() throws Exception {
-  }
-
   @After
   public void tearDown() throws Exception {
     if (hs2Conn != null) {
-      try {
-        hs2Conn.close();
-      } catch (Exception e) {
-        // Ignore shutdown errors since there are negative tests
-      }
+      hs2Conn.close();
+      hs2Conn = null;
+    }
+    if (miniHS2 != null && miniHS2.isStarted()) {
+      miniHS2.stop();
+      miniHS2.cleanup();
     }
-  }
-
-  @AfterClass
-  public static void afterTest() throws Exception {
-    miniHS2.stop();
   }
 
   @Test
@@ -100,9 +115,10 @@ public void testCookie() throws Exception {
     }
     stmt.execute("drop table " + tableName);
     stmt.close();
+
+    testCookieNegative();
   }
 
-  @Test
   public void testCookieNegative() throws Exception {
     try {
       // Trying to connect with a non-existent user should still fail with
@@ -115,6 +131,6 @@ public void testCookieNegative() throws Exception {
 
   private Connection getConnection(String userName) throws Exception {
     miniHiveKdc.loginUser(userName);
-    return new HiveConnection(miniHS2.getJdbcURL(), new Properties());
+    return new HiveConnection(miniHS2.getHttpJdbcURL(), new Properties());
   }
 }

itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.minikdc;
+
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+
+import org.junit.BeforeClass;
+
+/**
+ * JdbcWithMiniKdcSQLAuthTest for the case of hive.server2.transport.mode=all.
+ */
+public class TestJdbcWithMiniKdcSQLAuthAll extends JdbcWithMiniKdcSQLAuthTest {
+
+  @BeforeClass
+  public static void beforeTest() throws Exception {
+    JdbcWithMiniKdcSQLAuthTest.beforeTestBase(MiniHS2.HS2_ALL_MODE);
+  }
+}

itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java

@@ -47,7 +47,13 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+/**
+ * BeelineWithHS2ConnectionFileTestBase test.
+ */
+@RunWith(Parameterized.class)
 public abstract class BeelineWithHS2ConnectionFileTestBase {
   protected MiniHS2 miniHS2;
   protected HiveConf hiveConf = new HiveConf();
@@ -65,6 +71,10 @@ public abstract class BeelineWithHS2ConnectionFileTestBase {
 
   protected Map<String, String> confOverlay = new HashMap<>();
 
+  @Parameterized.Parameter
+  public String transportMode =  null;
+
+
   protected class TestBeeLine extends BeeLine {
     UserHS2ConnectionFileParser testHs2ConfigFileManager;
     ByteArrayOutputStream os;
@@ -163,7 +173,8 @@ public void before() throws Exception {
     miniHS2 = getNewMiniHS2();
     confOverlay = new HashMap<String, String>();
     confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
-    confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary");
+    confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, transportMode);
+    confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "false");
   }
 
   protected MiniHS2 getNewMiniHS2() throws Exception {

itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java

@@ -22,27 +22,52 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.Assume;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+/**
+ * TestBeelineConnectionUsingHiveSite test.
+ */
+@RunWith(Parameterized.class)
 public class TestBeelineConnectionUsingHiveSite extends BeelineWithHS2ConnectionFileTestBase {
+
+  boolean isHttpModeTest = false;
+
+  @Parameterized.Parameters(name = "{index}: tranportMode={0}")
+  public static Collection<Object[]> transportModes() {
+    return Arrays.asList(new Object[][]{{MiniHS2.HS2_BINARY_MODE}, {MiniHS2.HS2_HTTP_MODE}, {MiniHS2.HS2_ALL_MODE}});
+  }
   @Test
   public void testBeelineConnectionHttp() throws Exception {
+    Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_HTTP_MODE)
+        || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE));
+    isHttpModeTest = true;
     setupHs2();
     String path = createDefaultHs2ConnectionFile();
     assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName);
+    isHttpModeTest = false;
   }
 
   @Test
   public void testBeelineConnectionSSL() throws Exception {
+    Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE)
+        || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE));
     setupSSLHs2();
     String path = createDefaultHs2ConnectionFile();
     assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName);
   }
 
   @Test
   public void testBeelineConnectionNoAuth() throws Exception {
+    Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE)
+        || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE));
     setupNoAuthHs2();
     String path = createDefaultHs2ConnectionFile();
     assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName);
@@ -64,6 +89,8 @@ public void testBeelineDoesntUseDefaultIfU() throws Exception {
    */
   @Test
   public void testBeelineWithNoConnectionFile() throws Exception {
+    Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE)
+        || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE));
     setupNoAuthHs2();
     BeelineResult res = getBeelineOutput(null, new String[] {"-e", "show tables;" });
     assertEquals(1, res.exitCode);
@@ -72,6 +99,8 @@ public void testBeelineWithNoConnectionFile() throws Exception {
 
   @Test
   public void testBeelineUsingArgs() throws Exception {
+    Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE)
+        || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE));
     setupNoAuthHs2();
     String url = miniHS2.getBaseJdbcURL() + "default";
     String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo",
@@ -98,8 +127,6 @@ private void setupSSLHs2() throws Exception {
   }
 
   private void setupHs2() throws Exception {
-    confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE);
-    confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT);
     confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true");
     miniHS2.start(confOverlay);
     createTable();
@@ -108,6 +135,9 @@ private void setupHs2() throws Exception {
   private String createDefaultHs2ConnectionFile() throws Exception {
     Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter();
     String baseJdbcURL = miniHS2.getBaseJdbcURL();
+    if(isHttpModeTest) {
+      baseJdbcURL = miniHS2.getBaseHttpJdbcURL();
+    }
     System.out.println(baseJdbcURL);
     writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user",
         System.getProperty("user.name"));