/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.swift.http;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpHost;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.swift.auth.ApiKeyAuthenticationRequest;
import org.apache.hadoop.fs.swift.auth.ApiKeyCredentials;
import org.apache.hadoop.fs.swift.auth.AuthenticationRequest;
import org.apache.hadoop.fs.swift.auth.AuthenticationRequestWrapper;
import org.apache.hadoop.fs.swift.auth.AuthenticationResponse;
import org.apache.hadoop.fs.swift.auth.AuthenticationWrapper;
import org.apache.hadoop.fs.swift.auth.KeyStoneAuthRequest;
import org.apache.hadoop.fs.swift.auth.KeystoneApiKeyCredentials;
import org.apache.hadoop.fs.swift.auth.PasswordAuthenticationRequest;
import org.apache.hadoop.fs.swift.auth.PasswordCredentials;
import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
import org.apache.hadoop.fs.swift.auth.entities.Catalog;
import org.apache.hadoop.fs.swift.auth.entities.Endpoint;
import org.apache.hadoop.fs.swift.exceptions.SwiftAuthenticationFailedException;
import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.apache.hadoop.fs.swift.exceptions.SwiftException;
import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
import org.apache.hadoop.fs.swift.exceptions.SwiftThrottledRequestException;
import org.apache.hadoop.fs.swift.util.Duration;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.DurationStatsTable;
import org.apache.hadoop.fs.swift.util.JSONUtil;
import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import org.apache.http.conn.params.ConnRoutePNames;

import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.List;
import java.util.Properties;

import static org.apache.commons.httpclient.HttpStatus.*;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;

/**
 * This implements the client-side of the Swift REST API
 *
 * The core actions put, get and query data in the Swift object store,
 * after authenticationg the client.
 *
 * <b>Logging:</b>
 *
 * Logging at DEBUG level displays detail about the actions of this
 * client, including HTTP requests and responses -excluding authentication
 * details.
 */
public final class SwiftRestClient {
  private static final Log LOG = LogFactory.getLog(SwiftRestClient.class);

  /**
   * Header that says "use newest version" -ensures that
   * the query doesn't pick up older versions served by
   * an eventually consistent filesystem (except in the special case
   * of a network partition, at which point no guarantees about
   * consistency can be made.
   */
  public static final Header NEWEST =
          new Header(SwiftProtocolConstants.X_NEWEST, "true");

  /**
   * the authentication endpoint as supplied in the configuration
   */
  private final URI authUri;

  /**
   * Swift region. Some OpenStack installations has more than one region.
   * In this case user can specify the region with which Hadoop will be working
   */
  private final String region;

  /**
   * tenant name
   */
  private final String tenant;

  /**
   * username name
   */
  private final String username;

  /**
   * user password
   */
  private final String password;

  /**
   * user api key
   */
  private final String apiKey;

  /**
   * The authentication request used to authenticate with Swift
   */
  private final AuthenticationRequest authRequest;

  /**
   * This auth request is similar to @see authRequest,
   * with one difference: it has another json representation when
   * authRequest one is not applicable
   */
  private AuthenticationRequest keystoneAuthRequest;

  private boolean useKeystoneAuthentication = false;

    /**
   * The container this client is working with
   */
  private final String container;
  private final String serviceDescription;

  /**
   * Access token (Secret)
   */
  private AccessToken token;

  /**
   * Endpoint for swift operations, obtained after authentication
   */
  private URI endpointURI;

  /**
   * URI under which objects can be found.
   * This is set when the user is authenticated -the URI
   * is returned in the body of the success response.
   */
  private URI objectLocationURI;

  private final URI filesystemURI;

  /**
   * The name of the service provider
   */
  private final String serviceProvider;

  /**
   * Should the public swift endpoint be used, rather than the in-cluster one?
   */
  private final boolean usePublicURL;

  /**
   * Number of times to retry a connection
   */
  private final int retryCount;

  /**
   * How long (in milliseconds) should a connection be attempted
   */
  private final int connectTimeout;

  /**
   * How long (in milliseconds) should a connection be attempted
   */
  private final int socketTimeout;

  /**
   * How long (in milliseconds) between bulk operations
   */
  private final int throttleDelay;

  /**
  * the name of a proxy host (can be null, in which case there is no proxy)
   */
  private String proxyHost;

  /**
   * The port of a proxy. This is ignored if {@link #proxyHost} is null
   */
  private int proxyPort;

  /**
   * Flag to indicate whether or not the client should
   * query for file location data.
   */
  private final boolean locationAware;

  private final int partSizeKB;
  /**
   * The blocksize of this FS
   */
  private final int blocksizeKB;
  private final int bufferSizeKB;

  private final DurationStatsTable durationStats = new DurationStatsTable();
  /**
   * objects query endpoint. This is synchronized
   * to handle a simultaneous update of all auth data in one
   * go.
   */
  private synchronized URI getEndpointURI() {
    return endpointURI;
  }

  /**
   * object location endpoint
   */
  private synchronized URI getObjectLocationURI() {
    return objectLocationURI;
  }

  /**
   * token for Swift communication
   */
  private synchronized AccessToken getToken() {
    return token;
  }

  /**
   * Setter of authentication and endpoint details.
   * Being synchronized guarantees that all three fields are set up together.
   * It is up to the reader to read all three fields in their own
   * synchronized block to be sure that they are all consistent.
   *
   * @param endpoint endpoint URI
   * @param objectLocation object location URI
   * @param authToken auth token
   */
  private void setAuthDetails(URI endpoint,
                              URI objectLocation,
                              AccessToken authToken) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("setAuth: endpoint=%s; objectURI=%s; token=%s",
              endpoint, objectLocation, authToken));
    }
    synchronized (this) {
      endpointURI = endpoint;
      objectLocationURI = objectLocation;
      token = authToken;
    }
  }


  /**
   * Base class for all Swift REST operations
   *
   * @param <M> method
   * @param <R> result
   */
  private static abstract class HttpMethodProcessor<M extends HttpMethod, R> {
    public final M createMethod(String uri) throws IOException {
      final M method = doCreateMethod(uri);
      setup(method);
      return method;
    }

    /**
     * Override it to return some result after method is executed.
     */
    public abstract R extractResult(M method) throws IOException;

    /**
     * Factory method to create a REST method against the given URI
     *
     * @param uri target
     * @return method to invoke
     */
    protected abstract M doCreateMethod(String uri);

    /**
     * Override port to set up the method before it is executed.
     */
    protected void setup(M method) throws IOException {
    }

    /**
     * Override point: what are the status codes that this operation supports
     *
     * @return an array with the permitted status code(s)
     */
    protected int[] getAllowedStatusCodes() {
      return new int[]{
              SC_OK,
              SC_CREATED,
              SC_ACCEPTED,
              SC_NO_CONTENT,
              SC_PARTIAL_CONTENT,
      };
    }
  }

  private static abstract class GetMethodProcessor<R> extends HttpMethodProcessor<GetMethod, R> {
    @Override
    protected final GetMethod doCreateMethod(String uri) {
      return new GetMethod(uri);
    }
  }

  private static abstract class PostMethodProcessor<R> extends HttpMethodProcessor<PostMethod, R> {
    @Override
    protected final PostMethod doCreateMethod(String uri) {
      return new PostMethod(uri);
    }
  }

  /**
   * There's a special type for auth messages, so that low-level
   * message handlers can react to auth failures differently from everything
   * else.
   */
  private static class AuthPostMethod extends PostMethod {


    private AuthPostMethod(String uri) {
      super(uri);
    }
  }

  /**
   * Generate an auth message
   * @param <R> response
   */
  private static abstract class AuthMethodProcessor<R> extends
                                                       HttpMethodProcessor<AuthPostMethod, R> {
    @Override
    protected final AuthPostMethod doCreateMethod(String uri) {
      return new AuthPostMethod(uri);
    }
  }

  private static abstract class PutMethodProcessor<R> extends HttpMethodProcessor<PutMethod, R> {
    @Override
    protected final PutMethod doCreateMethod(String uri) {
      return new PutMethod(uri);
    }

    /**
     * Override point: what are the status codes that this operation supports
     *
     * @return the list of status codes to accept
     */
    @Override
    protected int[] getAllowedStatusCodes() {
      return new int[]{
              SC_OK,
              SC_CREATED,
              SC_NO_CONTENT,
              SC_ACCEPTED,
      };
    }
  }

  /**
   * Create operation
   *
   * @param <R>
   */
  private static abstract class CopyMethodProcessor<R> extends HttpMethodProcessor<CopyMethod, R> {
    @Override
    protected final CopyMethod doCreateMethod(String uri) {
      return new CopyMethod(uri);
    }

    /**
     * The only allowed status code is 201:created
     * @return an array with the permitted status code(s)
     */
    @Override
    protected int[] getAllowedStatusCodes() {
      return new int[]{
              SC_CREATED
      };
    }
  }

  /**
   * Delete operation
   *
   * @param <R>
   */
  private static abstract class DeleteMethodProcessor<R> extends HttpMethodProcessor<DeleteMethod, R> {
    @Override
    protected final DeleteMethod doCreateMethod(String uri) {
      return new DeleteMethod(uri);
    }

    @Override
    protected int[] getAllowedStatusCodes() {
      return new int[]{
              SC_OK,
              SC_ACCEPTED,
              SC_NO_CONTENT,
              SC_NOT_FOUND
      };
    }
  }

  private static abstract class HeadMethodProcessor<R> extends HttpMethodProcessor<HeadMethod, R> {
    @Override
    protected final HeadMethod doCreateMethod(String uri) {
      return new HeadMethod(uri);
    }
  }


  /**
   * Create a Swift Rest Client instance.
   *
   * @param filesystemURI filesystem URI
   * @param conf The configuration to use to extract the binding
   * @throws SwiftConfigurationException the configuration is not valid for
   * defining a rest client against the service
   */
  private SwiftRestClient(URI filesystemURI,
                          Configuration conf)
      throws SwiftConfigurationException {
    this.filesystemURI = filesystemURI;
    Properties props = RestClientBindings.bind(filesystemURI, conf);
    String stringAuthUri = getOption(props, SWIFT_AUTH_PROPERTY);
    username = getOption(props, SWIFT_USERNAME_PROPERTY);
    password = props.getProperty(SWIFT_PASSWORD_PROPERTY);
    apiKey = props.getProperty(SWIFT_APIKEY_PROPERTY);
    //optional
    region = props.getProperty(SWIFT_REGION_PROPERTY);
    //tenant is optional
    tenant = props.getProperty(SWIFT_TENANT_PROPERTY);
    //service is used for diagnostics
    serviceProvider = props.getProperty(SWIFT_SERVICE_PROPERTY);
    container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
    String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
    usePublicURL = "true".equals(isPubProp);

        if (apiKey == null && password == null) {
            throw new SwiftConfigurationException(
                    "Configuration for " + filesystemURI +" must contain either "
                            + SWIFT_PASSWORD_PROPERTY + " or "
                            + SWIFT_APIKEY_PROPERTY);
        }
        //create the (reusable) authentication request
        if (password != null) {
            authRequest = new PasswordAuthenticationRequest(tenant,
                    new PasswordCredentials(
                            username,
                            password));
        } else {
            authRequest = new ApiKeyAuthenticationRequest(tenant,
                    new ApiKeyCredentials(
                            username, apiKey));
            keystoneAuthRequest = new KeyStoneAuthRequest(tenant,
                    new KeystoneApiKeyCredentials(username, apiKey));
    }
    locationAware = "true".equals(
      props.getProperty(SWIFT_LOCATION_AWARE_PROPERTY, "false"));

    //now read in properties that are shared across all connections

    //connection and retries
    try {
      retryCount = conf.getInt(SWIFT_RETRY_COUNT, DEFAULT_RETRY_COUNT);
      connectTimeout = conf.getInt(SWIFT_CONNECTION_TIMEOUT,
                                   DEFAULT_CONNECT_TIMEOUT);
      socketTimeout = conf.getInt(SWIFT_SOCKET_TIMEOUT,
                                   DEFAULT_SOCKET_TIMEOUT);

      throttleDelay = conf.getInt(SWIFT_THROTTLE_DELAY,
                                  DEFAULT_THROTTLE_DELAY);

      //proxy options
      proxyHost = conf.get(SWIFT_PROXY_HOST_PROPERTY);
      proxyPort = conf.getInt(SWIFT_PROXY_PORT_PROPERTY, 8080);

      blocksizeKB = conf.getInt(SWIFT_BLOCKSIZE,
                                DEFAULT_SWIFT_BLOCKSIZE);
      if (blocksizeKB <= 0) {
        throw new SwiftConfigurationException("Invalid blocksize set in "
                          + SWIFT_BLOCKSIZE
                          + ": " + blocksizeKB);
      }
      partSizeKB = conf.getInt(SWIFT_PARTITION_SIZE,
                               DEFAULT_SWIFT_PARTITION_SIZE);
      if (partSizeKB <=0) {
        throw new SwiftConfigurationException("Invalid partition size set in "
                                              + SWIFT_PARTITION_SIZE
                                              + ": " + partSizeKB);
      }

      bufferSizeKB = conf.getInt(SWIFT_REQUEST_SIZE,
                                 DEFAULT_SWIFT_REQUEST_SIZE);
      if (bufferSizeKB <=0) {
        throw new SwiftConfigurationException("Invalid buffer size set in "
                          + SWIFT_REQUEST_SIZE
                          + ": " + bufferSizeKB);
      }
    } catch (NumberFormatException e) {
      //convert exceptions raised parsing ints and longs into
      // SwiftConfigurationException instances
      throw new SwiftConfigurationException(e.toString(), e);
    }
    //everything you need for diagnostics. The password is omitted.
    serviceDescription = String.format(
      "Service={%s} container={%s} uri={%s}"
      + " tenant={%s} user={%s} region={%s}"
      + " publicURL={%b}"
      + " location aware={%b}"
      + " partition size={%d KB}, buffer size={%d KB}"
      + " block size={%d KB}"
      + " connect timeout={%d}, retry count={%d}"
      + " socket timeout={%d}"
      + " throttle delay={%d}"
      ,
      serviceProvider,
      container,
      stringAuthUri,
      tenant,
      username,
      region != null ? region : "(none)",
      usePublicURL,
      locationAware,
      partSizeKB,
      bufferSizeKB,
      blocksizeKB,
      connectTimeout,
      retryCount,
      socketTimeout,
      throttleDelay
      );
    if (LOG.isDebugEnabled()) {
      LOG.debug(serviceDescription);
    }
    try {
      this.authUri = new URI(stringAuthUri);
    } catch (URISyntaxException e) {
      throw new SwiftConfigurationException("The " + SWIFT_AUTH_PROPERTY
              + " property was incorrect: "
              + stringAuthUri, e);
    }
  }

  /**
   * Get a mandatory configuration option
   *
   * @param props property set
   * @param key   key
   * @return value of the configuration
   * @throws SwiftConfigurationException if there was no match for the key
   */
  private static String getOption(Properties props, String key) throws
          SwiftConfigurationException {
    String val = props.getProperty(key);
    if (val == null) {
      throw new SwiftConfigurationException("Undefined property: " + key);
    }
    return val;
  }

  /**
   * Make an HTTP GET request to Swift to get a range of data in the object.
   *
   * @param path   path to object
   * @param offset offset from file beginning
   * @param length file length
   * @return The input stream -which must be closed afterwards.
   * @throws IOException Problems
   * @throws SwiftException swift specific error
   * @throws FileNotFoundException path is not there
   */
  public HttpBodyContent getData(SwiftObjectPath path,
                                 long offset,
                                 long length) throws IOException {
    if (offset < 0) {
      throw new SwiftException("Invalid offset: " + offset
                            + " in getDataAsInputStream( path=" + path
                            + ", offset=" + offset
                            + ", length =" + length + ")");
    }
    if (length <= 0) {
      throw new SwiftException("Invalid length: " + length
                + " in getDataAsInputStream( path="+ path
                            + ", offset=" + offset
                            + ", length ="+ length + ")");
    }

    final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
            offset,
            offset + length - 1);
    if (LOG.isDebugEnabled()) {
      LOG.debug("getData:" + range);
    }

    return getData(path,
                   new Header(HEADER_RANGE, range),
                   SwiftRestClient.NEWEST);
  }

  /**
   * Returns object length
   *
   * @param uri file URI
   * @return object length
   * @throws SwiftException on swift-related issues
   * @throws IOException on network/IO problems
   */
  public long getContentLength(URI uri) throws IOException {
    preRemoteCommand("getContentLength");
    return perform("getContentLength", uri, new HeadMethodProcessor<Long>() {
      @Override
      public Long extractResult(HeadMethod method) throws IOException {
        return method.getResponseContentLength();
      }

      @Override
      protected void setup(HeadMethod method) throws IOException {
        super.setup(method);
        method.addRequestHeader(NEWEST);
      }
    });
  }

  /**
   * Get the length of the remote object
   * @param path object to probe
   * @return the content length
   * @throws IOException on any failure
   */
  public long getContentLength(SwiftObjectPath path) throws IOException {
    return getContentLength(pathToURI(path));
  }

  /**
   * Get the path contents as an input stream.
   * <b>Warning:</b> this input stream must be closed to avoid
   * keeping Http connections open.
   *
   * @param path path to file
   * @param requestHeaders http headers
   * @return byte[] file data or null if the object was not found
   * @throws IOException on IO Faults
   * @throws FileNotFoundException if there is nothing at the path
   */
  public HttpBodyContent getData(SwiftObjectPath path,
                                 final Header... requestHeaders)
          throws IOException {
    preRemoteCommand("getData");
    return doGet(pathToURI(path),
            requestHeaders);
  }

  /**
   * Returns object location as byte[]
   *
   * @param path path to file
   * @param requestHeaders http headers
   * @return byte[] file data or null if the object was not found
   * @throws IOException on IO Faults
   */
  public byte[] getObjectLocation(SwiftObjectPath path,
                                  final Header... requestHeaders) throws IOException {
    if (!isLocationAware()) {
      //if the filesystem is not location aware, do not ask for this information
      return null;
    }
    preRemoteCommand("getObjectLocation");
    try {
      return perform("getObjectLocation", pathToObjectLocation(path),
              new GetMethodProcessor<byte[]>() {
                @Override
                protected int[] getAllowedStatusCodes() {
                  return new int[]{
                    SC_OK,
                    SC_FORBIDDEN,
                    SC_NO_CONTENT
                  };
                }

                @Override
                public byte[] extractResult(GetMethod method) throws
                        IOException {

                  //TODO: remove SC_NO_CONTENT if it depends on Swift versions
                  if (method.getStatusCode() == SC_NOT_FOUND
                      || method.getStatusCode() == SC_FORBIDDEN ||
                          method.getStatusCode() == SC_NO_CONTENT ||
                          method.getResponseBodyAsStream() == null) {
                    return null;
                  }
                  final InputStream responseBodyAsStream = method.getResponseBodyAsStream();
                  final byte[] locationData = new byte[1024];

                  return responseBodyAsStream.read(locationData) > 0 ? locationData : null;
                }

                @Override
                     protected void setup(GetMethod method)
                       throws SwiftInternalStateException {
                  setHeaders(method, requestHeaders);
                }
              });
    } catch (IOException e) {
      LOG.warn("Failed to get the location of " + path + ": " + e, e);
      return null;
    }
  }

  /**
   * Create the URI needed to query the location of an object
   * @param path object path to retrieve information about
   * @return the URI for the location operation
   * @throws SwiftException if the URI could not be constructed
   */
  private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException {
    URI uri;
    String dataLocationURI = objectLocationURI.toString();
    try {
      if (path.toString().startsWith("/")) {
        dataLocationURI = dataLocationURI.concat(path.toUriPath());
      } else {
        dataLocationURI = dataLocationURI.concat("/").concat(path.toUriPath());
      }

      uri = new URI(dataLocationURI);
    } catch (URISyntaxException e) {
      throw new SwiftException(e);
    }
    return uri;
  }

  /**
   * Find objects under a prefix
   *
   * @param path path prefix
   * @param requestHeaders optional request headers
   * @return byte[] file data or null if the object was not found
   * @throws IOException on IO Faults
   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
   * the directory is empty
   */
  public byte[] findObjectsByPrefix(SwiftObjectPath path,
                                    final Header... requestHeaders) throws IOException {
    preRemoteCommand("findObjectsByPrefix");
    URI uri;
    String dataLocationURI = getEndpointURI().toString();
    try {
      String object = path.getObject();
      if (object.startsWith("/")) {
        object = object.substring(1);
      }
      object = encodeUrl(object);
      dataLocationURI = dataLocationURI.concat("/")
              .concat(path.getContainer())
              .concat("/?prefix=")
              .concat(object)
      ;
      uri = new URI(dataLocationURI);
    } catch (URISyntaxException e) {
      throw new SwiftException("Bad URI: " + dataLocationURI, e);
    }

    return perform("findObjectsByPrefix", uri, new GetMethodProcessor<byte[]>() {
      @Override
      public byte[] extractResult(GetMethod method) throws IOException {
        if (method.getStatusCode() == SC_NOT_FOUND) {
          //no result
          throw new FileNotFoundException("Not found " + method.getURI());
        }
        return method.getResponseBody();
      }

      @Override
      protected int[] getAllowedStatusCodes() {
        return new int[]{
                SC_OK,
                SC_NOT_FOUND
        };
      }

      @Override
      protected void setup(GetMethod method) throws
                     SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Find objects in a directory
   *
   * @param path path prefix
   * @param requestHeaders optional request headers
   * @return byte[] file data or null if the object was not found
   * @throws IOException on IO Faults
   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
   * the directory is empty
   */
  public byte[] listDeepObjectsInDirectory(SwiftObjectPath path,
                                           boolean listDeep,
                                       final Header... requestHeaders)
          throws IOException {
    preRemoteCommand("listDeepObjectsInDirectory");

    String endpoint = getEndpointURI().toString();
    StringBuilder dataLocationURI = new StringBuilder();
    dataLocationURI.append(endpoint);
    String object = path.getObject();
    if (object.startsWith("/")) {
      object = object.substring(1);
    }
    if (!object.endsWith("/")) {
      object = object.concat("/");
    }

    if (object.equals("/")) {
      object = "";
    }

    dataLocationURI = dataLocationURI.append("/")
            .append(path.getContainer())
            .append("/?prefix=")
            .append(object)
            .append("&format=json");

    //in listing deep set param to false
    if (listDeep == false) {
        dataLocationURI.append("&delimiter=/");
    }

    return findObjects(dataLocationURI.toString(), requestHeaders);
  }

  /**
   * Find objects in a location
   * @param location URI
   * @param requestHeaders optional request headers
   * @return the body of te response
   * @throws IOException IO problems
   */
  private byte[] findObjects(String location, final Header[] requestHeaders) throws
          IOException {
    URI uri;
    preRemoteCommand("findObjects");
    try {
      uri = new URI(location);
    } catch (URISyntaxException e) {
      throw new SwiftException("Bad URI: " + location, e);
    }

    return perform("findObjects", uri, new GetMethodProcessor<byte[]>() {
      @Override
      public byte[] extractResult(GetMethod method) throws IOException {
        if (method.getStatusCode() == SC_NOT_FOUND) {
          //no result
          throw new FileNotFoundException("Not found " + method.getURI());
        }
        return method.getResponseBody();
      }

      @Override
      protected int[] getAllowedStatusCodes() {
        return new int[]{
                SC_OK,
                SC_NOT_FOUND
        };
      }

      @Override
      protected void setup(GetMethod method)
        throws SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Copy an object. This is done by sending a COPY method to the filesystem
   * which is required to handle this WebDAV-level extension to the
   * base HTTP operations.
   *
   * @param src source path
   * @param dst destination path
   * @param headers any headers
   * @return true if the status code was considered successful
   * @throws IOException on IO Faults
   */
  public boolean copyObject(SwiftObjectPath src, final SwiftObjectPath dst,
                            final Header... headers) throws IOException {

    preRemoteCommand("copyObject");

    return perform("copy", pathToURI(src), new CopyMethodProcessor<Boolean>() {
      @Override
      public Boolean extractResult(CopyMethod method) throws IOException {
        return method.getStatusCode() != SC_NOT_FOUND;
      }

      @Override
      protected void setup(CopyMethod method) throws
                     SwiftInternalStateException {
        setHeaders(method, headers);
        method.addRequestHeader(HEADER_DESTINATION, dst.toUriPath());
      }
    });
  }

  /**
   * Uploads file as Input Stream to Swift.
   * The data stream will be closed after the request.
   *
   * @param path path to Swift
   * @param data object data
   * @param length length of data
   * @param requestHeaders http headers
   * @throws IOException on IO Faults
   */
  public void upload(SwiftObjectPath path,
                     final InputStream data,
                     final long length,
                     final Header... requestHeaders)
          throws IOException {
    preRemoteCommand("upload");

    try {
      perform("upload", pathToURI(path), new PutMethodProcessor<byte[]>() {
        @Override
        public byte[] extractResult(PutMethod method) throws IOException {
          return method.getResponseBody();
        }

        @Override
        protected void setup(PutMethod method) throws
                        SwiftInternalStateException {
          method.setRequestEntity(new InputStreamRequestEntity(data, length));
          setHeaders(method, requestHeaders);
        }
      });
    } finally {
      data.close();
    }

  }


  /**
   * Deletes object from swift.
   * The result is true if this operation did the deletion.
   *
   * @param path           path to file
   * @param requestHeaders http headers
   * @throws IOException on IO Faults
   */
  public boolean delete(SwiftObjectPath path, final Header... requestHeaders) throws IOException {
    preRemoteCommand("delete");

    return perform("", pathToURI(path), new DeleteMethodProcessor<Boolean>() {
      @Override
      public Boolean extractResult(DeleteMethod method) throws IOException {
        return method.getStatusCode() == SC_NO_CONTENT;
      }

      @Override
      protected void setup(DeleteMethod method) throws
                    SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Issue a head request
   * @param reason reason -used in logs
   * @param path path to query
   * @param requestHeaders request header
   * @return the response headers. This may be an empty list
   * @throws IOException IO problems
   * @throws FileNotFoundException if there is nothing at the end
   */
  public Header[] headRequest(String reason,
                              SwiftObjectPath path,
                              final Header... requestHeaders)
          throws IOException {

    preRemoteCommand("headRequest: "+ reason);
    return perform(reason, pathToURI(path), new HeadMethodProcessor<Header[]>() {
      @Override
      public Header[] extractResult(HeadMethod method) throws IOException {
        if (method.getStatusCode() == SC_NOT_FOUND) {
          throw new FileNotFoundException("Not Found " + method.getURI());
        }

        return method.getResponseHeaders();
      }

      @Override
      protected void setup(HeadMethod method) throws
                    SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Issue a put request
   * @param path path
   * @param requestHeaders optional headers
   * @return the HTTP response
   * @throws IOException any problem
   */
  public int putRequest(SwiftObjectPath path, final Header... requestHeaders)
          throws IOException {

    preRemoteCommand("putRequest");
    return perform(pathToURI(path), new PutMethodProcessor<Integer>() {

      @Override
      public Integer extractResult(PutMethod method) throws IOException {
        return method.getStatusCode();
      }

      @Override
      protected void setup(PutMethod method) throws
                    SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Authenticate to Openstack Keystone
   * As well as returning the access token, the member fields {@link #token},
   * {@link #endpointURI} and {@link #objectLocationURI} are set up for re-use.
   * <p>
   * This method is re-entrant -if more than one thread attempts to authenticate
   * neither will block -but the field values with have those of the last caller.
   *
   * @return authenticated access token
   */
  public AccessToken authenticate() throws IOException {
    final AuthenticationRequest authenticationRequest;
    if (useKeystoneAuthentication) {
      authenticationRequest = keystoneAuthRequest;
    } else {
      authenticationRequest = authRequest;
    }

    LOG.debug("started authentication");
    return perform("authentication",
                   authUri,
                   new AuthenticationPost(authenticationRequest));
  }

  private class AuthenticationPost extends AuthMethodProcessor<AccessToken> {
    final AuthenticationRequest authenticationRequest;

    private AuthenticationPost(AuthenticationRequest authenticationRequest) {
      this.authenticationRequest = authenticationRequest;
    }

    @Override
    protected void setup(AuthPostMethod method) throws IOException {

      method.setRequestEntity(getAuthenticationRequst(authenticationRequest));
    }

    /**
     * specification says any of the 2xxs are OK, so list all
     * the standard ones
     * @return a set of 2XX status codes.
     */
    @Override
    protected int[] getAllowedStatusCodes() {
      return new int[]{
        SC_OK,
        SC_BAD_REQUEST,
        SC_CREATED,
        SC_ACCEPTED,
        SC_NON_AUTHORITATIVE_INFORMATION,
        SC_NO_CONTENT,
        SC_RESET_CONTENT,
        SC_PARTIAL_CONTENT,
        SC_MULTI_STATUS,
        SC_UNAUTHORIZED //if request unauthorized, try another method
      };
    }

    @Override
    public AccessToken extractResult(AuthPostMethod method) throws IOException {
      //initial check for failure codes leading to authentication failures
      if (method.getStatusCode() == SC_BAD_REQUEST) {
        throw new SwiftAuthenticationFailedException(
          authenticationRequest.toString(), "POST", authUri, method);
      }

      final AuthenticationResponse access =
        JSONUtil.toObject(method.getResponseBodyAsString(),
                          AuthenticationWrapper.class).getAccess();
      final List<Catalog> serviceCatalog = access.getServiceCatalog();
      //locate the specific service catalog that defines Swift; variations
      //in the name of this add complexity to the search
      boolean catalogMatch = false;
      StringBuilder catList = new StringBuilder();
      StringBuilder regionList = new StringBuilder();

      //these fields are all set together at the end of the operation
      URI endpointURI = null;
      URI objectLocation;
      Endpoint swiftEndpoint = null;
      AccessToken accessToken;

      for (Catalog catalog : serviceCatalog) {
        String name = catalog.getName();
        String type = catalog.getType();
        String descr = String.format("[%s: %s]; ", name, type);
        catList.append(descr);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Catalog entry " + descr);
        }
        if (name.equals(SERVICE_CATALOG_SWIFT)
            || name.equals(SERVICE_CATALOG_CLOUD_FILES)
            || type.equals(SERVICE_CATALOG_OBJECT_STORE)) {
          //swift is found
          if (LOG.isDebugEnabled()) {
            LOG.debug("Found swift catalog as " + name + " => " + type);
          }
          //now go through the endpoints
          for (Endpoint endpoint : catalog.getEndpoints()) {
            String endpointRegion = endpoint.getRegion();
            URI publicURL = endpoint.getPublicURL();
            URI internalURL = endpoint.getInternalURL();
            descr = String.format("[%s => %s / %s]; ",
                                  endpointRegion,
                                  publicURL,
                                  internalURL);
            regionList.append(descr);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Endpoint " + descr);
            }
            if (region == null || endpointRegion.equals(region)) {
              endpointURI = usePublicURL ? publicURL : internalURL;
              swiftEndpoint = endpoint;
              break;
            }
          }
        }
      }
      if (endpointURI == null) {
        String message = "Could not find swift service from auth URL "
                         + authUri
                         + " and region '" + region + "'. "
                         + "Categories: " + catList
                         + ((regionList.length() > 0) ?
                            ("regions: " + regionList)
                                                      : "No regions");
        throw new SwiftInvalidResponseException(message,
                                                SC_OK,
                                                "authenticating",
                                                authUri);

      }


      accessToken = access.getToken();
      String path = SWIFT_OBJECT_AUTH_ENDPOINT
                    + swiftEndpoint.getTenantId();
      String host = endpointURI.getHost();
      try {
        objectLocation = new URI(endpointURI.getScheme(),
                                 null,
                                 host,
                                 endpointURI.getPort(),
                                 path,
                                 null,
                                 null);
      } catch (URISyntaxException e) {
        throw new SwiftException("object endpoint URI is incorrect: "
                                 + endpointURI
                                 + " + " + path,
                                 e);
      }
      setAuthDetails(endpointURI, objectLocation, accessToken);

      if (LOG.isDebugEnabled()) {
        LOG.debug("authenticated against " + endpointURI);
      }
      createDefaultContainer();
      return accessToken;
    }
  }

  private StringRequestEntity getAuthenticationRequst(AuthenticationRequest authenticationRequest)
    throws IOException {
    final String data = JSONUtil.toJSON(new AuthenticationRequestWrapper(
            authenticationRequest));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Authenticating with " + authenticationRequest);
    }
    return toJsonEntity(data);
  }

  /**
   * create default container if it doesn't exist for Hadoop Swift integration.
   * non-reentrant, as this should only be needed once.
   *
   * @throws IOException IO problems.
   */
  private synchronized void createDefaultContainer() throws IOException {
    createContainer(container);
  }

  /**
   * Create a container -if it already exists, do nothing
   *
   * @param containerName the container name
   * @throws IOException IO problems
   * @throws SwiftBadRequestException invalid container name
   * @throws SwiftInvalidResponseException error from the server
   */
  public void createContainer(String containerName) throws IOException {
    SwiftObjectPath objectPath = new SwiftObjectPath(containerName, "");
    try {
      //see if the data is there
      headRequest("createContainer", objectPath, NEWEST);
    } catch (FileNotFoundException ex) {
      int status = 0;
      try {
        status = putRequest(objectPath);
      } catch (FileNotFoundException e) {
        //triggered by a very bad container name.
        //re-insert the 404 result into the status
        status = SC_NOT_FOUND;
      }
      if (status == SC_BAD_REQUEST) {
        throw new SwiftBadRequestException(
          "Bad request -authentication failure or bad container name?",
          status,
          "PUT",
          null);
      }
      if (!isStatusCodeExpected(status,
              SC_OK,
              SC_CREATED,
              SC_ACCEPTED,
              SC_NO_CONTENT)) {
        throw new SwiftInvalidResponseException("Couldn't create container "
                + containerName +
                " for storing data in Swift." +
                " Try to create container " +
                containerName + " manually ",
                status,
                "PUT",
                null);
      } else {
        throw ex;
      }
    }
  }

  /**
   * Trigger an initial auth operation if some of the needed
   * fields are missing
   *
   * @throws IOException on problems
   */
  private void authIfNeeded() throws IOException {
    if (getEndpointURI() == null) {
      authenticate();
    }
  }

  /**
   * Pre-execution actions to be performed by methods. Currently this
   * <ul>
   *   <li>Logs the operation at TRACE</li>
   *   <li>Authenticates the client -if needed</li>
   * </ul>
   * @throws IOException
   */
  private void preRemoteCommand(String operation) throws IOException {
    if (LOG.isTraceEnabled()) {
      LOG.trace("Executing " + operation);
    }
    authIfNeeded();
  }


  /**
   * Performs the HTTP request, validates the response code and returns
   * the received data. HTTP Status codes are converted into exceptions.
   *
   * @param uri URI to source
   * @param processor HttpMethodProcessor
   * @param <M> method
   * @param <R> result type
   * @return result of HTTP request
   * @throws IOException IO problems
   * @throws SwiftBadRequestException the status code indicated "Bad request"
   * @throws SwiftInvalidResponseException the status code is out of range
   * for the action (excluding 404 responses)
   * @throws SwiftInternalStateException the internal state of this client
   * is invalid
   * @throws FileNotFoundException a 404 response was returned
   */
  private <M extends HttpMethod, R> R perform(URI uri,
                      HttpMethodProcessor<M, R> processor)
    throws IOException,
           SwiftBadRequestException,
           SwiftInternalStateException,
           SwiftInvalidResponseException,
           FileNotFoundException {
    return perform("",uri, processor);
  }

  /**
   * Performs the HTTP request, validates the response code and returns
   * the received data. HTTP Status codes are converted into exceptions.
   * @param reason why is this operation taking place. Used for statistics
   * @param uri URI to source
   * @param processor HttpMethodProcessor
   * @param <M> method
   * @param <R> result type
   * @return result of HTTP request
   * @throws IOException IO problems
   * @throws SwiftBadRequestException the status code indicated "Bad request"
   * @throws SwiftInvalidResponseException the status code is out of range
   * for the action (excluding 404 responses)
   * @throws SwiftInternalStateException the internal state of this client
   * is invalid
   * @throws FileNotFoundException a 404 response was returned
   */
  private <M extends HttpMethod, R> R perform(String reason,
                                              URI uri,
                                              HttpMethodProcessor<M, R> processor)
      throws IOException, SwiftBadRequestException, SwiftInternalStateException,
            SwiftInvalidResponseException, FileNotFoundException {
    checkNotNull(uri);
    checkNotNull(processor);

    final M method = processor.createMethod(uri.toString());

    //retry policy
    HttpMethodParams methodParams = method.getParams();
    methodParams.setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(
                    retryCount, false));
    methodParams.setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT,
                                 connectTimeout);
    methodParams.setSoTimeout(socketTimeout);
    method.addRequestHeader(HEADER_USER_AGENT, SWIFT_USER_AGENT);
    Duration duration = new Duration();
    boolean success = false;
    try {
      int statusCode = 0;
      try {
        statusCode = exec(method);
      } catch (IOException e) {
        //rethrow with extra diagnostics and wiki links
        throw ExceptionDiags.wrapException(uri.toString(), method.getName(), e);
      }

      //look at the response and see if it was valid or not.
      //Valid is more than a simple 200; even 404 "not found" is considered
      //valid -which it is for many methods.

      //validate the allowed status code for this operation
      int[] allowedStatusCodes = processor.getAllowedStatusCodes();
      boolean validResponse = isStatusCodeExpected(statusCode,
              allowedStatusCodes);

      if (!validResponse) {
        IOException ioe = buildException(uri, method, statusCode);
        throw ioe;
      }

      R r = processor.extractResult(method);
      success = true;
      return r;
    } catch (IOException e) {
      //release the connection -always
      method.releaseConnection();
      throw e;
    } finally {
      duration.finished();
      durationStats.add(method.getName()+" " + reason, duration, success);
    }
  }

  /**
   * Build an exception from a failed operation. This can include generating
   * specific exceptions (e.g. FileNotFound), as well as the default
   * {@link SwiftInvalidResponseException}.
   *
   * @param uri URI for operation
   * @param method operation that failed
   * @param statusCode status code
   * @param <M> method type
   * @return an exception to throw
   */
  private <M extends HttpMethod> IOException buildException(URI uri,
                                                            M method,
                                                            int statusCode) {
    IOException fault;

    //log the failure @debug level
    String errorMessage = String.format("Method %s on %s failed, status code: %d," +
            " status line: %s",
            method.getName(),
            uri,
            statusCode,
            method.getStatusLine()
    );
    if (LOG.isDebugEnabled()) {
      LOG.debug(errorMessage);
    }
    //send the command
    switch (statusCode) {
      case SC_NOT_FOUND:
        fault = new FileNotFoundException("Operation " + method.getName()
                + " on " + uri);
        break;

      case SC_BAD_REQUEST:
        //bad HTTP request
        fault =  new SwiftBadRequestException(
          "Bad request against " + uri,
          method.getName(),
          uri,
          method);
        break;

      case SC_REQUESTED_RANGE_NOT_SATISFIABLE:
        //out of range
        StringBuilder errorText = new StringBuilder(method.getStatusText());
        //get the requested length
        Header requestContentLen = method.getRequestHeader(HEADER_CONTENT_LENGTH);
        if (requestContentLen!=null) {
          errorText.append(" requested ").append(requestContentLen.getValue());
        }
        //and the result
        Header availableContentRange = method.getResponseHeader(
          HEADER_CONTENT_RANGE);
        if (requestContentLen!=null) {
          errorText.append(" available ").append(availableContentRange.getValue());
        }
        fault = new EOFException(errorText.toString());
        break;

      case SC_UNAUTHORIZED:
        //auth failure; should only happen on the second attempt
        fault  = new SwiftAuthenticationFailedException(
                        "Operation not authorized- current access token ="
                            + getToken(),
                        method.getName(),
                        uri,
                        method);
        break;

      case SwiftProtocolConstants.SC_TOO_MANY_REQUESTS_429:
      case SwiftProtocolConstants.SC_THROTTLED_498:
        //response code that may mean the client is being throttled
        fault  = new SwiftThrottledRequestException(
                        "Client is being throttled: too many requests",
                        method.getName(),
                        uri,
                        method);
        break;

      default:
        //return a generic invalid HTTP response
        fault = new SwiftInvalidResponseException(
                errorMessage,
                method.getName(),
                uri,
                method);
    }

    return fault;
  }

  /**
   * Exec a GET request and return the input stream of the response
   *
   * @param uri URI to GET
   * @param requestHeaders request headers
   * @return the input stream. This must be closed to avoid log errors
   * @throws IOException
   */
  private HttpBodyContent doGet(final URI uri, final Header... requestHeaders) throws IOException {
    return perform("", uri, new GetMethodProcessor<HttpBodyContent>() {
      @Override
      public HttpBodyContent extractResult(GetMethod method) throws IOException {
        return
          new HttpBodyContent(
            new HttpInputStreamWithRelease(uri, method), method.getResponseContentLength()
          );
      }

      @Override
      protected void setup(GetMethod method) throws
                    SwiftInternalStateException {
        setHeaders(method, requestHeaders);
      }
    });
  }

  /**
   * Create an instance against a specific FS URI,
   *
   * @param filesystemURI filesystem to bond to
   * @param config source of configuration data
   * @return REST client instance
   * @throws IOException on instantiation problems
   */
  public static SwiftRestClient getInstance(URI filesystemURI,
                                            Configuration config) throws IOException {
    return new SwiftRestClient(filesystemURI, config);
  }


  /**
   * Convert the (JSON) data to a string request as UTF-8
   *
   * @param data data
   * @return the data
   * @throws SwiftException if for some very unexpected reason it's impossible
   * to convert the data to UTF-8.
   */
  private static StringRequestEntity toJsonEntity(String data) throws
          SwiftException {
    StringRequestEntity entity;
    try {
      entity = new StringRequestEntity(data, "application/json", "UTF-8");
    } catch (UnsupportedEncodingException e) {
      throw new SwiftException("Could not encode data as UTF-8", e);
    }
    return entity;
  }

  /**
   * Converts Swift path to URI to make request.
   * This is public for unit testing
   *
   * @param path path to object
   * @param endpointURI damain url e.g. http://domain.com
   * @return valid URI for object
   * @throws SwiftException
   */
  public static URI pathToURI(SwiftObjectPath path,
                              URI endpointURI) throws SwiftException {
    checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");

    String dataLocationURI = endpointURI.toString();
    try {

      dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
      return new URI(dataLocationURI);
    } catch (URISyntaxException e) {
      throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
    }
  }

  /**
   * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
   * with a replacement of + with %20.
   * @param url URL string
   * @return an encoded string
   * @throws SwiftException if the URL cannot be encoded
   */
  private static String encodeUrl(String url) throws SwiftException {
    if (url.matches(".*\\s+.*")) {
      try {
        url = URLEncoder.encode(url, "UTF-8");
        url = url.replace("+", "%20");
      } catch (UnsupportedEncodingException e) {
        throw new SwiftException("failed to encode URI", e);
      }
    }

    return url;
  }

  /**
   * Convert a swift path to a URI relative to the current endpoint.
   *
   * @param path path
   * @return an path off the current endpoint URI.
   * @throws SwiftException
   */
  private URI pathToURI(SwiftObjectPath path) throws SwiftException {
    return pathToURI(path, getEndpointURI());
  }

  /**
   * Add the headers to the method, and the auth token (which must be set
   * @param method method to update
   * @param requestHeaders the list of headers
   * @throws SwiftInternalStateException not yet authenticated
   */
  private void setHeaders(HttpMethodBase method, Header[] requestHeaders)
      throws SwiftInternalStateException {
      for (Header header : requestHeaders) {
        method.addRequestHeader(header);
      }
    setAuthToken(method, getToken());
  }


  /**
   * Set the auth key header of the method to the token ID supplied
   *
   * @param method method
   * @param accessToken access token
   * @throws SwiftInternalStateException if the client is not yet authenticated
   */
  private void setAuthToken(HttpMethodBase method, AccessToken accessToken)
      throws SwiftInternalStateException {
    checkNotNull(accessToken,"Not authenticated");
    method.addRequestHeader(HEADER_AUTH_KEY, accessToken.getId());
  }

  /**
   * Execute a method in a new HttpClient instance.
   * If the auth failed, authenticate then retry the method.
   *
   * @param method methot to exec
   * @param <M> Method type
   * @return the status code
   * @throws IOException on any failure
   */
  private <M extends HttpMethod> int exec(M method) throws IOException {
    final HttpClient client = new HttpClient();
    if (proxyHost != null) {
      client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
              new HttpHost(proxyHost, proxyPort));
    }

    int statusCode = execWithDebugOutput(method, client);

    if ((statusCode == HttpStatus.SC_UNAUTHORIZED
            || statusCode == HttpStatus.SC_BAD_REQUEST)
            && method instanceof AuthPostMethod
            && !useKeystoneAuthentication) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Operation failed with status " + method.getStatusCode() +
                 " attempting keystone auth");
      }
      //if rackspace key authentication failed - try custom Keystone authentication
      useKeystoneAuthentication = true;
      final AuthPostMethod authentication = (AuthPostMethod) method;
      //replace rackspace auth with keystone one
      authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest));
      statusCode = execWithDebugOutput(method, client);
    }

    if (statusCode == HttpStatus.SC_UNAUTHORIZED ) {
      //unauthed -or the auth uri rejected it.

      if (method instanceof AuthPostMethod) {
          //unauth response from the AUTH URI itself.
          throw new SwiftAuthenticationFailedException(authRequest.toString(),
                                                       "auth",
                                                       authUri,
                                                       method);
      }
      //any other URL: try again
      if (LOG.isDebugEnabled()) {
        LOG.debug("Reauthenticating");
      }
      //re-auth, this may recurse into the same dir
      authenticate();
      if (LOG.isDebugEnabled()) {
        LOG.debug("Retrying original request");
      }
      statusCode = execWithDebugOutput(method, client);
    }
    return statusCode;
  }

  /**
   * Execute the request with the request and response logged at debug level
   * @param method method to execute
   * @param client client to use
   * @param <M> method type
   * @return the status code
   * @throws IOException any failure reported by the HTTP client.
   */
  private <M extends HttpMethod> int execWithDebugOutput(M method,
                                                         HttpClient client) throws
          IOException {
    if (LOG.isDebugEnabled()) {
      StringBuilder builder = new StringBuilder(
              method.getName() + " " + method.getURI() + "\n");
      for (Header header : method.getRequestHeaders()) {
        builder.append(header.toString());
      }
      LOG.debug(builder);
    }
    int statusCode = client.executeMethod(method);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Status code = " + statusCode);
    }
    return statusCode;
  }

  /**
   * Ensures that an object reference passed as a parameter to the calling
   * method is not null.
   *
   * @param reference an object reference
   * @return the non-null reference that was validated
   * @throws NullPointerException if {@code reference} is null
   */
  private static <T> T checkNotNull(T reference) throws
            SwiftInternalStateException {
    return checkNotNull(reference, "Null Reference");
  }

  private static <T> T checkNotNull(T reference, String message) throws
            SwiftInternalStateException {
    if (reference == null) {
      throw new SwiftInternalStateException(message);
    }
    return reference;
  }

  /**
   * Check for a status code being expected -takes a list of expected values
   *
   * @param status received status
   * @param expected expected value
   * @return true iff status is an element of [expected]
   */
  private boolean isStatusCodeExpected(int status, int... expected) {
    for (int code : expected) {
      if (status == code) {
        return true;
      }
    }
    return false;
  }


  @Override
  public String toString() {
    return "Swift client: " + serviceDescription;
  }

  /**
   * Get the region which this client is bound to
   * @return the region
   */
  public String getRegion() {
    return region;
  }

  /**
   * Get the tenant to which this client is bound
   * @return the tenant
   */
  public String getTenant() {
    return tenant;
  }

  /**
   * Get the username this client identifies itself as
   * @return the username
   */
  public String getUsername() {
    return username;
  }

  /**
   * Get the container to which this client is bound
   * @return the container
   */
  public String getContainer() {
    return container;
  }

  /**
   * Is this client bound to a location aware Swift blobstore
   * -that is, can you query for the location of partitions
   * @return true iff the location of multipart file uploads
   * can be determined.
   */
  public boolean isLocationAware() {
    return locationAware;
  }

  /**
   * Get the blocksize of this filesystem
   * @return a blocksize &gt; 0
   */
  public long getBlocksizeKB() {
    return blocksizeKB;
  }

  /**
   * Get the partition size in KB
   * @return the partition size
   */
  public int getPartSizeKB() {
    return partSizeKB;
  }

  /**
   * Get the buffer size in KB
   * @return the buffer size wanted for reads
   */
  public int getBufferSizeKB() {
    return bufferSizeKB;
  }

  public int getProxyPort() {
    return proxyPort;
  }

  public String getProxyHost() {
    return proxyHost;
  }

  public int getRetryCount() {
    return retryCount;
  }

  public int getConnectTimeout() {
    return connectTimeout;
  }

  public boolean isUsePublicURL() {
    return usePublicURL;
  }

  public int getThrottleDelay() {
    return throttleDelay;
  }

  /**
   * Get the current operation statistics
   * @return a snapshot of the statistics
   */

  public List<DurationStats> getOperationStatistics() {
    return durationStats.getDurationStatistics();
  }
}
