WiP: Thrift upgrade

beeline/pom.xml

@@ -133,6 +133,16 @@
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpcore</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>net.sf.supercsv</groupId>

cli/pom.xml

@@ -88,6 +88,16 @@
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpcore</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java

@@ -84,7 +84,7 @@ public void testThriftMaxMessageSize() throws Throwable {
     });
     String exceptionMessage = expectedException.getMessage();
     // Verify the Thrift library is enforcing the limit
-    assertTrue(exceptionMessage.contains("MaxMessageSize reached"));
+    assertTrue(exceptionMessage.contains("Message size exceeds limit"));
     limitedClient.close();
 
     // test default client (with a default THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE)

itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java

@@ -114,7 +114,7 @@ public void testHmsThriftMaxMessageSize() throws Exception {
     });
     String exceptionMessage = expectedException.getMessage();
     // Verify the Thrift library is enforcing the limit
-    assertTrue(exceptionMessage.contains("MaxMessageSize reached"));
+    assertTrue(exceptionMessage.contains("Message size exceeds limit"));
     limitedClient.close();
 
     MetastoreConf.setVar(clientConf, MetastoreConf.ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE, "1048576000");

itests/hive-unit/pom.xml

@@ -113,6 +113,11 @@
       <groupId>org.apache.tez</groupId>
       <artifactId>tez-protobuf-history-plugin</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
+      <version>${httpcomponents5.client.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.tez</groupId>
       <artifactId>tez-tests</artifactId>

itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHs2ConnectionMetricsHttp.java

@@ -24,14 +24,14 @@
 import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
 import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hc.client5.http.cookie.CookieStore;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
 import org.apache.hive.jdbc.HttpBasicAuthInterceptor;
 import org.apache.hive.service.rpc.thrift.TCLIService;
 import org.apache.hive.service.rpc.thrift.TCloseSessionReq;
 import org.apache.hive.service.rpc.thrift.TOpenSessionReq;
 import org.apache.hive.service.rpc.thrift.TOpenSessionResp;
 import org.apache.hive.service.rpc.thrift.TSessionHandle;
-import org.apache.http.client.CookieStore;
-import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.transport.THttpClient;
@@ -89,14 +89,14 @@ public void testOpenConnectionMetrics() throws Exception {
   }
 
   private TCLIService.Client getHttpClient() throws Exception {
-    DefaultHttpClient httpClient = new DefaultHttpClient();
+    HttpClientBuilder httpClient = HttpClientBuilder.create();
 
     Map<String, String> headers = new HashMap<>();
     headers.put("Connection", "close");
-    httpClient.addRequestInterceptor(new BasicHttpRequestInterceptor(USERNAME, PASSWORD, null,
+    httpClient.addRequestInterceptorLast(new BasicHttpRequestInterceptor(USERNAME, PASSWORD, null,
             null, false, headers));
 
-    TTransport transport = new THttpClient(getHttpUrl(), httpClient);
+    TTransport transport = new THttpClient(getHttpUrl(), httpClient.build());
     TProtocol protocol = new TBinaryProtocol(transport);
     return new TCLIService.Client(protocol);
   }

itests/hive-unit/src/test/java/org/apache/hive/service/TestHttpServices.java

@@ -20,11 +20,11 @@
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.Header;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.Header;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -83,7 +83,7 @@ private void testHttpServiceDoesNotContainServerVersionAndXPoweredBy(String mini
       HttpGet request = new HttpGet(miniHS2);
 
       try (CloseableHttpResponse response = httpClient.execute(request)) {
-        for (Header header : response.getAllHeaders()) {
+        for (Header header : response.getHeaders()) {
           Assert.assertNotEquals("x-powered-by", header.getName().toLowerCase());
           Assert.assertNotEquals("server", header.getName().toLowerCase());
         }

itests/hive-unit/src/test/java/org/apache/hive/service/auth/TestHttpCookieAuthenticationTest.java

@@ -33,12 +33,12 @@
 import java.util.Map;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hc.client5.http.cookie.BasicCookieStore;
+import org.apache.hc.client5.http.cookie.Cookie;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.cookie.BasicClientCookie;
 import org.apache.hive.jdbc.HiveConnection;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
-import org.apache.http.client.CookieStore;
-import org.apache.http.client.HttpClient;
-import org.apache.http.cookie.Cookie;
-import org.apache.http.impl.cookie.BasicClientCookie;
 import org.apache.thrift.transport.THttpClient;
 import org.apache.thrift.transport.TTransport;
 import org.junit.AfterClass;
@@ -79,7 +79,7 @@ public void testHttpJdbcCookies() throws Exception {
     try(Connection connection = DriverManager.getConnection(miniHS2.getJdbcURL(), username, "bar")) {
       assertNotNull(connection);
 
-      CookieStore cookieStore = getCookieStoreFromConnection(connection);
+      BasicCookieStore cookieStore = getCookieStoreFromConnection(connection);
       assertNotNull(cookieStore);
 
       // Test that basic cookies worked
@@ -157,8 +157,8 @@ public void testHttpJdbcCookies() throws Exception {
   }
 
   // ((InternalHttpClient) ((THttpClient) ((HiveConnection) connection).transport).client).cookieStore.getCookies()
-  private CookieStore getCookieStoreFromConnection(Connection connection) throws Exception {
-    CookieStore cookieStore = null;
+  private BasicCookieStore getCookieStoreFromConnection(Connection connection) throws Exception {
+    BasicCookieStore cookieStore = null;
     if (connection instanceof HiveConnection) {
       HiveConnection hiveConnection = (HiveConnection) connection;
 
@@ -170,11 +170,12 @@ private CookieStore getCookieStoreFromConnection(Connection connection) throws E
         THttpClient httpTransport = (THttpClient) transport;
         Field clientField = httpTransport.getClass().getDeclaredField("client");
         clientField.setAccessible(true);
-        HttpClient httpClient = (HttpClient) clientField.get(httpTransport);
+
+        CloseableHttpClient httpClient = (CloseableHttpClient) clientField.get(httpTransport);
 
         Field cookieStoreField = httpClient.getClass().getDeclaredField("cookieStore");
         cookieStoreField.setAccessible(true);
-        cookieStore = (CookieStore) cookieStoreField.get(httpClient);
+        cookieStore = (BasicCookieStore) cookieStoreField.get(httpClient);
       }
     }
     return cookieStore;

itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java

@@ -45,20 +45,22 @@
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext;
 import org.apache.hadoop.hive.common.IPStackUtils;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.cookie.CookieStore;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.client5.http.protocol.RequestDefaultHeaders;
+import org.apache.hc.core5.http.EntityDetails;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.protocol.HttpContext;
 import org.apache.hive.jdbc.HttpBasicAuthInterceptor;
 import org.apache.hive.service.auth.HiveAuthConstants;
 import org.apache.hive.service.rpc.thrift.TCLIService;
 import org.apache.hive.service.rpc.thrift.TExecuteStatementReq;
 import org.apache.hive.service.rpc.thrift.TOpenSessionReq;
 import org.apache.hive.service.rpc.thrift.TOpenSessionResp;
-import org.apache.http.Header;
-import org.apache.http.HttpException;
-import org.apache.http.HttpRequest;
-import org.apache.http.client.CookieStore;
-import org.apache.http.client.protocol.RequestDefaultHeaders;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.protocol.HttpContext;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.transport.THttpClient;
@@ -101,13 +103,13 @@ public HttpBasicAuthInterceptorWithLogging(String username, String password,
     }
 
     @Override
-    public void process(HttpRequest httpRequest, HttpContext httpContext)
+    public void process(HttpRequest httpRequest, EntityDetails entityDetails, HttpContext httpContext)
         throws HttpException, IOException {
-      super.process(httpRequest, httpContext);
+      super.process(httpRequest, entityDetails, httpContext);
 
       String currHeaders = "";
 
-      for (org.apache.http.Header h : httpRequest.getAllHeaders()) {
+      for (Header h : httpRequest.getHeaders()) {
         currHeaders += h.getName() + ":" + h.getValue() + " ";
       }
       requestHeaders.add(currHeaders);
@@ -222,11 +224,15 @@ private TTransport getRawBinaryTransport() throws Exception {
   }
 
   private static TTransport getHttpTransport() throws Exception {
-    DefaultHttpClient httpClient = new DefaultHttpClient();
     String httpUrl = getHttpUrl();
-    httpClient.addRequestInterceptor(
-        new HttpBasicAuthInterceptor(USERNAME, PASSWORD,
-            null, null, false, null, null));
+
+    HttpClient httpClient = HttpClientBuilder.create()
+        .addRequestInterceptorLast(
+            new HttpBasicAuthInterceptor(USERNAME, PASSWORD,
+                null, null, false, null, null)
+        )
+        .build();
+
     return new THttpClient(httpUrl, httpClient);
   }
 
@@ -281,12 +287,13 @@ public void testCustomCookies() throws Exception {
   public HttpBasicAuthInterceptorWithLogging openSessionWithTestInterceptor(
           Map<String, String> additionalHeaders, Map<String, String> cookieHeaders) throws Exception {
     TTransport transport;
-    DefaultHttpClient hClient = new DefaultHttpClient();
     String httpUrl = getHttpUrl();
     HttpBasicAuthInterceptorWithLogging authInt =
-      new HttpBasicAuthInterceptorWithLogging(USERNAME, PASSWORD, null, null,
-      false, additionalHeaders, cookieHeaders);
-    hClient.addRequestInterceptor(authInt);
+        new HttpBasicAuthInterceptorWithLogging(USERNAME, PASSWORD, null, null,
+            false, additionalHeaders, cookieHeaders);
+    HttpClient hClient = HttpClientBuilder.create()
+        .addRequestInterceptorLast(authInt)
+        .build();
     transport = new THttpClient(httpUrl, hClient);
     TCLIService.Client httpClient = getClient(transport);
 
@@ -324,24 +331,24 @@ public void testForwardedHeaders() throws Exception {
 
   private void verifyForwardedHeaders(ArrayList<String> headerIPs, String cmd) throws Exception {
     TTransport transport;
-    DefaultHttpClient hClient = new DefaultHttpClient();
     String httpUrl = getHttpUrl();
+    HttpClientBuilder hClient = HttpClientBuilder.create();
 
     // add an interceptor that adds the X-Forwarded-For header with given ips
     if (!headerIPs.isEmpty()) {
       Header xForwardHeader = new BasicHeader("X-Forwarded-For", Joiner.on(",").join(headerIPs));
       RequestDefaultHeaders headerInterceptor = new RequestDefaultHeaders(
           Arrays.asList(xForwardHeader));
-      hClient.addRequestInterceptor(headerInterceptor);
+      hClient.addRequestInterceptorLast(headerInterceptor);
     }
 
     // interceptor for adding username, pwd
     HttpBasicAuthInterceptor authInt = new HttpBasicAuthInterceptor(USERNAME,
             PASSWORD, null, null,
         false, null, null);
-    hClient.addRequestInterceptor(authInt);
+    hClient.addRequestInterceptorLast(authInt);
 
-    transport = new THttpClient(httpUrl, hClient);
+    transport = new THttpClient(httpUrl, hClient.build());
     TCLIService.Client httpClient = getClient(transport);
 
     // Create a new open session request object

jdbc/pom.xml

@@ -113,16 +113,28 @@
     </dependency>
     <!-- inter-project -->
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
+      <version>${httpcomponents5.client.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpcore</artifactId>
+      <groupId>org.apache.httpcomponents.core5</groupId>
+      <artifactId>httpcore5</artifactId>
+      <version>${httpcomponents5.core.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpcore</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
@@ -179,6 +191,12 @@
       <artifactId>junit-vintage-engine</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <version>3.1</version>
+      <scope>compile</scope>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>

jdbc/src/java/org/apache/hive/jdbc/CsrfHttpRequestInterceptor.java

@@ -18,10 +18,11 @@
 
 package org.apache.hive.jdbc;
 
-import org.apache.http.HttpException;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.protocol.HttpContext;
+import org.apache.hc.core5.http.EntityDetails;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.HttpRequestInterceptor;
+import org.apache.hc.core5.http.protocol.HttpContext;
 
 import java.io.IOException;
 
@@ -47,10 +48,10 @@ public static void setInjectHeader(boolean headerInjection){
   }
 
   @Override
-  public void process(HttpRequest httpRequest, HttpContext httpContext)
+  public void process(HttpRequest request, EntityDetails entity, HttpContext context)
       throws HttpException, IOException {
     if (injectHeader){
-      httpRequest.addHeader("X-CSRF-TOKEN", "true");
+      request.addHeader("X-CSRF-TOKEN", "true");
     }
   }
 }