HTTP clients in/for Java

Russell Bateman
January 2018
last update:

This is hastily excerpted code from a project and is incomplete. I use it only for reference and comparison. (See also here.)

We're going to write code for three different APIs:

  1. HttpURLConnection, the ancient JDK client
  2. HttpClient, the Apache client, said to be several times faster
  3. OkHttpClient, a much more modern and understandable API

Let's pretend we need a client that always posts one kind of data and queries that kind of data from a server via fairly static URLs. We wrote the code for these three, different clients in a way to be able to switch between them for benchmarking.

HttpClientOperations.java:
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

public interface HttpClientOperations
{
  Integer QUERY_TIMEOUT  = 1000;                // milliseconds
  String  ACCEPT_CHARSET = "Accept-Charset";
  String  UTF8           = "UTF-8";

  String      postRecord( final String record ) throws IOException;
  InputStream getQuery  ( final String query )  throws IOException;
}

HttpURLConnection

HttpUrlOperations.java:
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

import org.slf4j.Logger;

public class HttpUrlOperations implements HttpClientOperations
{
  private HttpURLConnection connection;

  private static Logger logger;
  private static String postUrlString;  // something like "http://localhost:30000/cache/addrecords?store=0"
  private static URL    postURL;

  public static void injectPostUrl( final String url ) { postUrlString = url; }
  public static void injectLogger( Logger aLogger )    { logger = aLogger; }

  private static class OperationsHolder
  {
    private static final HttpUrlOperations INSTANCE = new HttpUrlOperations( logger);
  }

  public static HttpUrlOperations getInstance()
  {
    return OperationsHolder.INSTANCE;
  }

  /**
   * The URL is for POST operations. Note that
   * connection.setRequestProperty("Connection", "Keep-Alive");
   * ...is, in fact, the default so we don't need to make it explicit.
   *
   * @param logger if known.
   */
  private HttpUrlOperations( Logger logger )
  {
    HttpUrlOperations.logger = logger;
    try
    {
      postURL = new URL( postUrlString );
    }
    catch( IOException e )
    {
      postURL = null;
    }

    logInfo( "Using OkHttpClient client" );
  }

  /**
   * This posts data being written to the server.
   */
  public String postRecord( final String record ) throws IOException
  {
    if( postURL == null )
      throw new IOException( "URL to post to was botched at initialization" );

    connection = ( HttpURLConnection ) ( postURL.openConnection() );

    connection.setRequestMethod( "POST" );
    connection.setUseCaches( false );
    connection.setDoOutput( true );
    connection.setRequestProperty( "Accept", "application/xml,application/json" );
    connection.addRequestProperty( "Content-type", "application/xml" );

    try
    {
      DataOutputStream writer = new DataOutputStream( connection.getOutputStream() );
      writer.writeBytes( record );
      writer.close();
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred posting record (" + t.getMessage() + ')';
      throw new IOException( message );
    }

    // This is where the request is actually sent. No matter what,
    // get the response either to return or drain it.
    InputStream inputStream = connection.getInputStream();

    if( inputStream != null )
    {
      try
      {
        return StreamUtilities.fromStream( inputStream );
      }
      catch( IOException e )
      {
        Throwable t = e.getCause();
        String message = e.getMessage() + " occurred copying response (" + t.getMessage() + ')';
        throw new IOException( message );
      }
      finally
      {
        inputStream.close();
      }
    }

    InputStream errorStream = connection.getErrorStream();

    try
    {
      // the stream must be read through, so we do that.
      if( inputStream != null )
      {
        StreamUtilities.drainStream( inputStream, logger );
        inputStream.close();
      }

      // do similarly for the error stream.
      if( errorStream != null )
      {
        StreamUtilities.drainStream( errorStream, logger );
        errorStream.close();
      }
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred draining response and/or error (" + t.getMessage() + ')';
      throw new IOException( message );
    }
    finally
    {
      if( inputStream != null )
        inputStream.close();
      if( errorStream != null )
        errorStream.close();
    }

    return null;
  }


  /**
   * This gets data from the server.
   * @param query something like "http://localhost:30000/query?q=" plus the query
   * @return the server's response as an InputStream.
   * @throws IOException upon failure.
   */
  public InputStream getQuery( final String query ) throws IOException
  {
    connection = ( HttpURLConnection ) ( query.openConnection() );

    connection.setRequestProperty( ACCEPT_CHARSET, UTF8 );
    connection.setReadTimeout( QUERY_TIMEOUT );
    connection.setRequestMethod( "GET" );
    connection.setUseCaches( false );
    connection.setDoOutput( true );
    connection.setRequestProperty( "Accept", "application/xml,application/json" );

    try
    {
      return connection.getInputStream();
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred during query (" + t.getMessage() + ')';
      throw new IOException( message );
    }
  }

  public void injectLogging( Logger logger ) { HttpUrlOperations.logger = logger; }

  private void logInfo( final String info )
  {
    if( logger != null )
      logger.info( info );
  }
}

Apache HttpClient

ApacheClientOperations.java:
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.slf4j.Logger;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

public class ApacheClientOperations implements HttpClientOperations
{
  private static Logger              logger;
  private static CloseableHttpClient client;
  private static String              postURL; // something like "http://localhost:30000/cache/addrecords?store=0"

  public static void injectPostUrl( final String url ) { postURL = url; }
  public static void injectLogger( Logger aLogger )    { logger = aLogger; }

  private static class OperationsHolder
  {
    private static final ApacheClientOperations INSTANCE = new ApacheClientOperations( logger);
  }

  public static ApacheClientOperations getInstance()
  {
    return OperationsHolder.INSTANCE;
  }

  /**
   * The URL is for POST operations.
   *
   * @param logger if known.
   */
  private ApacheClientOperations( Logger logger )
  {
    ApacheClientOperations.logger = logger;
    ApacheClientOperations.client = HttpClients.createDefault();
    logInfo( "Using Apache HTTP client" );
  }

  /**
   * This posts data being written to the server.
   */
  @Override
  public String postRecord( String record ) throws IOException
  {
    HttpPost post = new HttpPost( postURL );

    List< NameValuePair > properties = new ArrayList<>();
    properties.add( new BasicNameValuePair( "Accept",         "application/xml,application/json" ) );
    properties.add( new BasicNameValuePair( "Accept-Charset", "charset=utf-8" ) );
    properties.add( new BasicNameValuePair( "Content-type",   "application/xml" ) );
    post.setEntity( new UrlEncodedFormEntity( properties ) );

    CloseableHttpResponse response = client.execute( post );
    HttpEntity            entity   = response.getEntity();

    try
    {
      EntityUtils.consume( entity );
      return null;
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred discarding response (" + t.getMessage() + ')';
      throw new IOException( message );
    }
    finally
    {
      response.close();
    }
  }

  /**
   * This gets data from the server. The query is built as a URL before calling.
   * @param query something like "http://localhost:30000/query?q=" plus the query
   * @return the server's response as an InputStream.
   * @throws IOException upon failure.
   */
  @Override
  public InputStream getQuery( String query ) throws IOException
  {
    HttpGet get = new HttpGet( query );

    get.setHeader( ACCEPT_CHARSET, UTF8 );
    get.setHeader( "Accept", "application/xml,application/json" );

    CloseableHttpResponse response = client.execute( get );
    HttpEntity            entity   = response.getEntity();

    try
    {
      return entity.getContent();
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred getting query response (" + t.getMessage() + ')';
      throw new IOException( message );
    }
    finally
    {
      // TODO: Ha! this renders the InputStream we want to return invalid! Whatever shall we do?
      //response.close();
    }
  }

  public void injectLogging( Logger logger ) { ApacheClientOperations.logger = logger; }

  private void logInfo( final String info )
  {
    if( logger != null )
      logger.info( info );
  }
}

OkHttpClient

OkHttpOperations.java:
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.concurrent.TimeUnit;

import org.slf4j.Logger;

import okhttp3.Interceptor;
import okhttp3.OkHttpClient;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;

public class OkHttpOperations implements HttpClientOperations
{
  private static Logger       logger;
  private static OkHttpClient okHttpClient;
  private static String       postURL;        // something like "http://localhost:30000/cache/addrecords?store=0"

  public static void injectPostUrl( final String url ) { postURL = url; }
  public static void injectLogger( Logger aLogger )    { logger = aLogger; }

  private static class OperationsHolder
  {
    private static final OkHttpOperations INSTANCE = new OkHttpOperations( logger);
  }

  public static OkHttpOperations getInstance()
  {
    return OperationsHolder.INSTANCE;
  }

  /**
   * The URL is for POST operations.
   *
   * @param logger if known.
   */
  private OkHttpOperations( Logger logger )
  {
    OkHttpOperations.logger = logger;
    okHttpClient = new OkHttpClient.Builder()
                                   .addInterceptor( new DefaultContentTypeInterceptor( "application/xml" ) )
                                   .readTimeout( QUERY_TIMEOUT, TimeUnit.MILLISECONDS )
                                   .build();
    logInfo( "Using OkHttpClient client" );
  }

  private class DefaultContentTypeInterceptor implements Interceptor
  {
    private String contentType;

    public DefaultContentTypeInterceptor( final String contentType ) { this.contentType = contentType; }

    @Override
    public Response intercept( Interceptor.Chain chain ) throws IOException
    {
      Request originalRequest      = chain.request();
      Request requestWithUserAgent = originalRequest.newBuilder()
                                                    .header("Content-Type", contentType )
                                                    .build();

      return chain.proceed( requestWithUserAgent );
    }
  }

  /**
   * This posts data being written to the server.
   */
  public String postRecord( final String record ) throws IOException
  {
    Request request = new Request.Builder()
                                 .url( postURL )
                                 .post( RequestBody.create(
                                           MediaType.parse( "application/xml; charset=utf-8" ), record ) )
                                 .build();
    try
    {
      Response     response = okHttpClient.newCall( request ).execute();
      ResponseBody body     = response.body();

      return( body != null ) ? body.string() : null;
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred posting record (" + t.getMessage() + ')';
      throw new IOException( message );
    }
  }

  /**
   * This gets data from the server. The query is built as a URL before calling.
   * @param query something like "http://localhost:30000/query?q=" plus the query
   * @return the server's response as an InputStream.
   * @throws IOException upon failure.
   */
  public InputStream getQuery( final String query ) throws IOException
  {
    try
    {
      Request      request  = new Request.Builder().url( query ).get().build();
      Response     response = okHttpClient.newCall( request ).execute();
      ResponseBody body     = response.body();

      if( body == null )      // TODO: learn a lot more about OkHttpClient failures!
        throw new IOException( "No response or other failure" );

      return body.byteStream();
    }
    catch( IOException e )
    {
      Throwable t = e.getCause();
      String message = e.getMessage() + " occurred during query (" + t.getMessage() + ')';
      throw new IOException( message );
    }
  }

  public void injectLogging( Logger logger ) { OkHttpOperations.logger = logger; }

  private void logInfo( final String info )
  {
    if( logger != null )
      logger.info( info );
  }
}
StreamUtilities.java:

Implemenation of String-from-Stream and drainStream().

import org.slf4j.Logger;

import java.io.IOException;
import java.io.InputStream;

public class StreamUtilities
{
  public static String fromStream( InputStream inputStream ) throws IOException
  {
    int           ch;
    StringBuilder sb = new StringBuilder();

    while( ( ch = inputStream.read() ) != -1 )
      sb.append( ( char ) ch );

    return sb.toString();
  }

  private static final Integer DRAINBUFFERSIZE = 65536;

  private static byte[] drainBuffer = new byte[ DRAINBUFFERSIZE ];

  /**
   * Per https://stackoverflow.com/questions/4767553/safe-use-of-httpurlconnection
   * and https://stackoverflow.com/questions/9943351/httpsurlconnection-and-keep-alive,
   * in order for the HttpURLConnection to remain open for continued use,
   * the response must be completely drained otherwise the connection will have to
   * close.
   *
   * We figure that the underlying HttpURLConnection layer knows when to stop
   * reading and won't attempt to read too much.
   *
   * TODO: how to optimize this? I considered available() and skip(), but I hear
   * bad things about them.
   *
   * @param inputStream with crap in it we don't care about, but must drain away.
   */
  public static void drainStream( InputStream inputStream, Logger logger )
  {
    int  thisRead;
    long bytesRead = 0;
    try
    {
      while( ( thisRead = inputStream.read( drainBuffer ) ) != -1 )
        bytesRead += thisRead;
    }
    catch( IOException e )
    {
      if( logger != null )
        logger.warn( "Failed while draining stream after reading " + bytesRead + " bytes" );
    }
  }
}