package be.iminds.ilabt.jfed.lowlevel;

import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Server;
import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Service;
import be.iminds.ilabt.jfed.log.ApiCallDetails;
import be.iminds.ilabt.jfed.log.Logger;
import be.iminds.ilabt.jfed.lowlevel.connection.*;
import be.iminds.ilabt.jfed.lowlevel.connection.HttpConnection;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.util.*;
import org.apache.http.*;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.*;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DecompressingHttpClient;
import org.apache.xmlrpc.XmlRpcException;
import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientException;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.net.ssl.SSLException;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * AbstractApi
 */
public abstract class AbstractApi {
    protected static final ReplyConverter<String> STRING_REPLY_CONVERTER = resultValueObject -> {
        if (resultValueObject != null) {
            String value = resultValueObject.toString();
            if (!value.isEmpty())
                return value;
        }
        return null;
    };
    protected static final ReplyConverter<Boolean> BOOLEAN_REPLY_CONVERTER = resultValueObject -> {
        try {
            return (Boolean) resultValueObject;
        } catch (Exception ignored) {
            return false;
        }
    };
    private final static org.slf4j.Logger LOG = LoggerFactory.getLogger(AbstractApi.class);
    protected final JFedPreferences jFedPreferences;
    //    /**
//     * When in strict mode, an API call will fail when a response could not be fully parsed.
//     * This mode should be used by most client libraries.
//     *
//     * When not in strict mode, the API will log the BadGeniReplyExceptions, but will try to continue.
//     * No guarantees will be made on the resulting objects when not in strict mode.
//     * This mode is useful for compliance tests (like automated-testing and probe).
//     */
//    private final boolean strictMode;

    @Nonnull
    private RetrySettings retrySettings;
    private ApiInfo.Api api;
    private Logger logger;
    private be.iminds.ilabt.jfed.lowlevel.HttpCallDetails lastHttpResult;
    private be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails lastXmlRpcResult;
    private ApiCallReply lastApiCallReply;
    private boolean allowGzip = true;
    private boolean allowDeflate = true;
    private static final Pattern PATTERN_KEEP_ALIVE_TIMEOUT = Pattern.compile("timeout=([0-9]*)");

    /**
     * @param logger        the logger to use. May be null if no logger is used.
     * @param retrySettings retrySettings to use, or null to use global (static!) retry settings
     */
    public AbstractApi(@Nonnull Logger logger,
                       @Nonnull RetrySettings retrySettings,
                       @Nonnull ApiInfo.Api api,
                       @Nonnull JFedPreferences jFedPreferences/*,
                       boolean strictMode */) {
        //   this.strictMode = strictMode;
        assert logger != null;
        assert api != null;
        assert jFedPreferences != null;

        this.logger = logger;
        this.retrySettings = retrySettings;
        this.api = api;
        this.jFedPreferences = jFedPreferences;
    }

    /**
     * auto retrying when busy enabled
     */
    public AbstractApi(@Nonnull Logger logger,
                       @Nonnull ApiInfo.Api api,
                       @Nonnull JFedPreferences jFedPreferences) {
        this(logger, jFedPreferences.getRetrySettings(), api, jFedPreferences);
    }

  /*  protected boolean isStrictMode() {
        return strictMode;
    }                    */

    public static String apiSpecifiesNullableString(Object o) throws BadReplyGeniException {
        if (o == null) return null;
        if (o instanceof String)
            return (String) o;
        else
            throw new BadReplyGeniException("The API specified a String was expected here, but a " + o.getClass().getName() + " was found instead. value=" + o);
    }

    public static String apiSpecifiesNonNullString(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof String)
            return (String) o;
        else
            throw new BadReplyGeniException("The API specified a String was expected here, but a " + (o == null ? null
                    : o.getClass().getName()) + " was found instead. value=" + o);
    }

    @SuppressWarnings("unchecked")
    public static List<String> apiSpecifiesListOfString(Object o) throws BadReplyGeniException {
        return apiSpecifiesListOfT(String.class, o);
    }

    @SuppressWarnings("unchecked")
    @Nonnull
    public static <T> List<T> apiSpecifiesListOfT(Class<T> tClass, Object o) throws BadReplyGeniException {
        if (o != null && o instanceof Object[]) {
            Object[] oArray = (Object[]) o;
            List<T> result = new ArrayList<>(oArray.length);

            for (Object el : oArray)
                if (tClass.isInstance(el)) {
                    result.add((T) el);
                } else {
                    throw new BadReplyGeniException("The API specified a List of " + tClass.getName() + " was expected here, but at least one element was " +
                            (el == null ? null
                                    : el.getClass().getName()) + ". elementValue=" + el + " ListValue=" + o);
                }
            return result;
        } else
            throw new BadReplyGeniException("The API specified a List of " + tClass.getName() + " was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=\"" + o + "\"");
    }

    @SuppressWarnings("unchecked")
    public static Map<String, Object> apiSpecifiesMapStringToObject(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof Map) {
            for (Object entryO : ((Map) o).entrySet()) {
                Map.Entry entry = (Map.Entry) entryO;
                if (entry == null) throw new BadReplyGeniException("at least one element is null. o=" + o);
                if (!(entry.getKey() instanceof String))
                    throw new BadReplyGeniException("The API specified a Map mapping Strings to Objects was expected here, but at least one element has a key of type  " +
                            (entry.getKey() == null ? null : entry.getKey().getClass().getName()) +
                            ". keyValue=" + entry.getKey() + " MapValue=" + o);
            }
            return (Map<String, Object>) o;
        } else
            throw new BadReplyGeniException("The API specified a Map mapping Strings to Objects was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=\"" + o + "\"");
    }

    @SuppressWarnings("unchecked")
    @Nonnull
    public static Map<String, String> apiSpecifiesMapStringToString(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof Map) {
            for (Object entryO : ((Map) o).entrySet()) {
                Map.Entry entry = (Map.Entry) entryO;
                if (entry == null) throw new BadReplyGeniException("at least one element is null. o=" + o);
                if (entry.getKey() == null || !(entry.getKey() instanceof String))
                    throw new BadReplyGeniException("The API specified a Map mapping Strings to String was expected here, but at least one element has a key of type  " +
                            (entry.getKey() == null ? null
                                    : entry.getKey().getClass().getName()) + ". keyValue=" + entry.getKey() + " MapValue=" + o);
                if (entry.getValue() == null || !(entry.getValue() instanceof String))
                    throw new BadReplyGeniException("The API specified a Map mapping Strings to String was expected here, but at least one element has a value of type  " +
                            (entry.getValue() == null ? null
                                    : entry.getValue().getClass().getName()) + ". valueValue=" + entry.getKey() + " MapValue=" + o);
            }
            return (Map<String, String>) o;
        } else
            throw new BadReplyGeniException("The API specified a Map mapping Strings to String was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=" + o);
    }

    public static int apiSpecifiesInt(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof Integer)
            return (Integer) o;
        else
            throw new BadReplyGeniException("The API specified a Integer was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=" + o);
    }

    public static String apiSpecifiesString(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof String)
            return (String) o;
        else
            throw new BadReplyGeniException("The API specified a String was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=" + o);
    }
  public static Boolean apiSpecifiesBoolean(Object o) throws BadReplyGeniException {
        if (o != null && o instanceof Boolean)
            return (Boolean) o;
        else
            throw new BadReplyGeniException("The API specified a Boolean was expected here, but a " +
                    (o == null ? null : o.getClass().getName()) + " was found instead. value=" + o);
    }

    public static String apiSpecifiesStringInMap(Map<String, Object> ht, String key) throws BadReplyGeniException {
        if (ht.containsKey(key))
            try {
                return apiSpecifiesString(ht.get(key));
            } catch (BadReplyGeniException e) {
                throw new BadReplyGeniException("The API specified the dictionary to have a String value for key '" + key + "', but it is not valid", e);
            }
        else
            throw new BadReplyGeniException("The API specified the dictionary to have a String value for key '" + key + "', but it is not present");
    }


    public static GeniUrn apiSpecifiesGeniUrn(Object o) throws BadReplyGeniException {
        try {
            return new GeniUrn(apiSpecifiesString(o));
        } catch (GeniUrn.GeniUrnParseException e) {
            throw new BadReplyGeniException("The API specified a GeniUrn was expected here, but could not parse '" + o + "'");
        }
    }

    public static GeniUrn apiSpecifiesGeniUrnInMap(Map<String, Object> ht, String key) throws BadReplyGeniException {
        if (ht.containsKey(key))
            try {
                return apiSpecifiesGeniUrn(ht.get(key));
            } catch (BadReplyGeniException e) {
                throw new BadReplyGeniException("The API specified the dictionary to have a GeniUrn value for key '" + key + "', but it is not valid", e);
            }
        else
            throw new BadReplyGeniException("The API specified the dictionary to have a GeniUrn value for key '" + key + "', but it is not present");
    }

    public static Date apiSpecifiesDate(Object dateValue) throws BadReplyGeniException {
        if (dateValue instanceof Date)
            return (Date) dateValue;
        else if (dateValue instanceof String)
            try {
                return RFC3339Util.rfc3339StringToDate((String) dateValue);
            } catch (java.text.ParseException e) {
                throw new BadReplyGeniException("Could not parse date '" + dateValue + "'");
            }
        else
            throw new BadReplyGeniException("Expected a valid date");
    }

    public static Date apiSpecifiesDateInMap(Map<String, Object> ht, String key) throws BadReplyGeniException {
        if (ht.containsKey(key))
            try {
                return apiSpecifiesDate(ht.get(key));
            } catch (BadReplyGeniException e) {
                throw new BadReplyGeniException("The API specified the dictionary to have a RFC3339 Date value for key '" + key + "', but it is not valid", e);
            }
        else
            throw new BadReplyGeniException("The API specified the dictionary to have a RFC3339 Date value for key '" + key + "', but it is not present");
    }


    /**
     * see AMv3 specification
     */
    public static List<Map> createCredentialsListWithTypeAndVersion(List<AnyCredential> credentialList) {
        if (credentialList.contains(null))
            throw new IllegalArgumentException("Illegal arguments: no AnyCredential is allowed to be null in credentialList: " + credentialList);

        List<Map> credentialsList =
                credentialList.stream().map(AnyCredential::getGeniV3Map).collect(Collectors.toList());

        assert !credentialsList.contains(null); //useful assert when mocking credentials
        return credentialsList;
    }

    protected static Map<String, Object> makeMethodParameters(String name, Object val) {
        Map<String, Object> res = new HashMap<String, Object>();
        res.put(name, val);
        return res;
    }

    protected static Map<String, Object> makeMethodParameters(Object... args) {
        assert args.length % 2 == 0;

        Map<String, Object> res = new HashMap<>();
        for (int i = 0; i < args.length; i += 2) {
            res.put((String) args[i], args[i + 1]);
        }
        return res;
    }

    /**
     * @returns true if the provided Throwable (thrown by xmlRpcClient.execute) means the server has somehow timed out
     * false does not mean the server hasn't timed out, just that this hasn't been derived from the returned Exception.
     * <p/>
     * this looks at the causes of the Throwable as well to check if it is a timeout.
     */
    public static boolean isTimeout(Throwable e) {
        if (e == null) return false;

        if (isTimeoutHelper(e)) return true;

        //loop deeper into causes
        while (e.getCause() != null) {
            e = e.getCause();
            if (isTimeoutHelper(e)) return true;
        }

        return false;
    }

    protected static boolean isTimeoutHelper(Throwable e) {
        if (e == null) return false;

        if (e instanceof InterruptedIOException) {
            assert e instanceof ConnectTimeoutException || e instanceof SocketTimeoutException;

            return true;
        }

        if (e instanceof NoHttpResponseException) {
            return true;
        }

        if (e instanceof IOException && e.getMessage() != null) {
            //example error message, see FEDIBBTDEV-1559:
            //    Could not open channel (The server refused to open the channel (SSH_OPEN_CONNECT_FAILED, 'Connection timed out'))
            if (e.getMessage().toLowerCase().contains("Connection timed out".toLowerCase()))
                return true;
            if (e.getMessage().toLowerCase().contains("SSH_OPEN_CONNECT_FAILED".toLowerCase()))
                return true;
            if (e.getMessage().toLowerCase().contains("The server refused to open the channel".toLowerCase()))
                return true;
            if (e.getMessage().toLowerCase().contains("Could not open channel".toLowerCase()))
                return true;
        }

        if (e instanceof XmlRpcClientException && e.getMessage().equals("Unable to end request"))
            return true;

        return false;
    }

    @Nonnull
    protected static Map<String, Object> createOptionsMap(@Nullable Map<String, ?> extraOptions) {
        Map<String, Object> options = new HashMap<>();
        if (extraOptions != null)
            options.putAll(extraOptions);
        return options;
    }

    /**
     * @return a human readable name for the implemented API
     */
    abstract public String getName();

    public ApiInfo.Api getApi() {
        return api;
    }

    public Logger getLogger() {
        return logger;
    }

    public void setLogger(Logger logger) {
        this.logger = logger;
    }

    public RetrySettings getRetrySettings() {
        return retrySettings;
    }

    public void setRetrySettings(RetrySettings retrySettings) {
        if (retrySettings == null)
            this.retrySettings = jFedPreferences.getRetrySettings();
        else
            this.retrySettings = retrySettings;
    }

    /**
     * @param xmlRpcRes
     * @param reply          may be null
     * @param javaMethodName may be null
     * @param geniMethodName
     * @param con
     * @return
     */
    protected void log(XMLRPCCallDetails xmlRpcRes,
                       ApiCallReply reply,
                       String javaMethodName,
                       String geniMethodName,
                       SfaConnection con,
                       Map<String, Object> methodParameters) {
        log(xmlRpcRes, reply, javaMethodName, geniMethodName, con, methodParameters, null);
    }

    protected void log(XMLRPCCallDetails xmlRpcRes,
                       ApiCallReply reply,
                       String javaMethodName,
                       String geniMethodName,
                       SfaConnection con,
                       Map<String, Object> methodParameters, Throwable extraException) {
        lastHttpResult = xmlRpcRes;
        lastXmlRpcResult = xmlRpcRes;
        lastApiCallReply = reply;

        //hack related to FEDIBBTDEV-815. If this is not done, the first successful connection after a retry reverts
        // to the original connection, which is not correct. Logging is then wrong.
        // (but just logging, everything does work correctly)
        if (xmlRpcRes != null &&
                xmlRpcRes.getLastConnection() != null &&
                xmlRpcRes.getLastConnection() instanceof SfaConnection) {
            con = (SfaConnection) xmlRpcRes.getLastConnection();
        }

        if (logger == null)
            return;

        Service service = con.getDebugInfo() != null ? con.getDebugInfo().getService() : null;
//        assert authority != null; //not guaranteed anymore
        String serverUrl = con.getServerUrl();
        assert xmlRpcRes.getServerUrl() == null || con.getServerUrl().equals(xmlRpcRes.getServerUrl()) :
                "Wrongfully assumed that XML-RPC always uses base URL: " + xmlRpcRes.getServerUrl() + " != " + con.getServerUrl();
        String apiName = getName();
        String httpRequestLine;
        String httpRequestHeaders;
        String httpRequest;
        StatusLine httpStatusLine;
        String httpReplyHeaders;
        String httpReply;
        if (xmlRpcRes == null) {
            if (con.getXmlRpcTransportFactory() != null) {
                httpRequestLine = con.getXmlRpcTransportFactory().getHttpRequestLine();
                httpRequestHeaders = con.getXmlRpcTransportFactory().getHttpRequestHeaders();
                httpRequest = con.getXmlRpcTransportFactory().getHttpSentHistory();
                httpStatusLine = con.getXmlRpcTransportFactory().getHttpStatusLine();
                httpReplyHeaders = con.getXmlRpcTransportFactory().getHttpResponseHeaders();
                httpReply = con.getXmlRpcTransportFactory().getHttpReceivedHistory();
            } else {
                //the connection must be closed
                httpRequestLine = null;
                httpRequestHeaders = null;
                httpRequest = null;
                httpStatusLine = null;
                httpReplyHeaders = null;
                httpReply = null;
            }
        } else {
            httpRequestLine = xmlRpcRes.getRequestHttpRequestLine();
            httpRequestHeaders = xmlRpcRes.getRequestHttpHeaders();
            httpRequest = xmlRpcRes.getRequestHttpContent();
            httpStatusLine = xmlRpcRes.getResultHttpStatusLineObject();
            httpReplyHeaders = xmlRpcRes.getResultHttpHeaders();
            httpReply = xmlRpcRes.getResultHttpContent();
        }
        Throwable ex = xmlRpcRes == null ? null : xmlRpcRes.getException();
        if (ex == null) ex = extraException;
        ApiCallDetails res = new ApiCallDetails(service, service != null ? service.getServer() : null,
                con.getConnectionConfig(),
                serverUrl,
                con.getConnectionId() + "",
                apiName, javaMethodName, geniMethodName,
                reply,
                httpRequestLine,
                httpRequestHeaders,
                httpRequest,
                httpStatusLine == null ? null : httpStatusLine.toString(),
                httpReplyHeaders,
                httpReply,
                xmlRpcRes == null ? null : xmlRpcRes.getRequest(),
                xmlRpcRes == null ? null : xmlRpcRes.getResult(),
                methodParameters,
                xmlRpcRes == null ? null : xmlRpcRes.getStartTime(),
                xmlRpcRes == null ? null : xmlRpcRes.getStopTime(),
                ex
        );
        getLogger().fireResult(res);
    }

    /**
     * @param reply          may be null
     * @param javaMethodName may be null
     * @param con
     * @param exception      may be null
     * @return
     */
    protected void logNonSfa(HttpCallDetails httpCallDetails,
                             ApiCallReply reply,
                             String javaMethodName,
                             String apiMethodName,
                             JFedConnection con,
                             Map<String, Object> methodParameters,
                             Exception exception) {
        lastHttpResult = httpCallDetails;
        lastXmlRpcResult = null;
        lastApiCallReply = reply;

        if (logger == null)
            return;

        Service service = con.getDebugInfo() != null ? con.getDebugInfo().getService() : null;
//        assert authority != null; //not guaranteed anymore
        String serverBaseUrl = con.getServerUrl();
        String apiName = getName();

        ApiCallDetails res = new ApiCallDetails(
                service, service.getServer(),
                con.getConnectionConfig(),
                //important!  httpCallDetails url is not always the same as connection url! URLs can differ: one connection object has a socket that can handle multiple calls to different URLs
                httpCallDetails != null ? httpCallDetails.getServerUrl()
                        : con.getConnectionConfig().getServerUrlString(),
                con.getConnectionId() + "",
                apiName, javaMethodName, apiMethodName,
                reply,
                httpCallDetails == null ? null : httpCallDetails.getRequestHttpRequestLine(),
                httpCallDetails == null ? null : httpCallDetails.getRequestHttpHeaders(),
                httpCallDetails == null ? null : httpCallDetails.getRequestHttpContent(),
                httpCallDetails == null ? null : httpCallDetails.getResultHttpStatusLine(),
                httpCallDetails == null ? null : httpCallDetails.getResultHttpHeaders(),
                httpCallDetails == null ? null : httpCallDetails.getResultHttpContent(),
                null /*xmlrpc request*/,
                null /*xmlrpc reply*/,
                methodParameters,
                httpCallDetails == null ? null : httpCallDetails.getStartTime(),
                httpCallDetails == null ? null : httpCallDetails.getStopTime(),
                exception
        );
        getLogger().fireResult(res);
    }

    public HttpCallDetails getLastHttpResult() {
        return lastHttpResult;
    }

    public be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails getLastXmlRpcResult() {
        return lastXmlRpcResult;
    }

    protected abstract boolean isBusyReply(be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails res);

    protected boolean isPossibleTemporaryErrorReply(be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails res) {
        return false;
    }

    protected boolean isPossibleTimeoutReply(be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails res) {
        return false;
    }

    protected HttpCallDetails executeHttpCall(HttpConnection con, String apiMethodName, HttpVerb httpVerb, String url,
                                              ContentType requestContentType, String requestContent, String javaMethodName,
                                              Map<String, Object> methodParameters,
                                              Map<String, Object> urlParameters, Map<String, String> extraHeaders) throws JFedException, MalformedURLException {
        return executeHttpCall(con, apiMethodName, httpVerb, url,
                requestContent == null ? null : new StringEntity(requestContent, requestContentType),
                javaMethodName, methodParameters, urlParameters, extraHeaders);
    }

    protected HttpCallDetails executeHttpCall(HttpConnection con, String apiMethodName, HttpVerb httpVerb, String url,
                                              ContentType requestContentType, String requestContent, String javaMethodName,
                                              Map<String, Object> methodParameters, Map<String, Object> urlParameters) throws JFedException, MalformedURLException {
        return executeHttpCall(con, apiMethodName, httpVerb, url,
                requestContent == null ? null : new StringEntity(requestContent, requestContentType),
                javaMethodName, methodParameters, urlParameters, null/*extraHeaders*/);
    }

    protected void setAllowGzip(boolean allowGzip) {
        this.allowGzip = allowGzip;
    }

    public void setAllowDeflate(boolean allowDeflate) {
        this.allowDeflate = allowDeflate;
    }

    protected HttpCallDetails executeHttpCall(HttpConnection con, String apiMethodName, HttpVerb httpVerb, String baseUrl,
                                              HttpEntity requestEntity, String javaMethodName,
                                              Map<String, Object> methodParameters,
                                              Map<String, Object> urlParameters, Map<String, String> extraHeaders) throws JFedException, MalformedURLException {
        assert con != null;
        ConnectionConfig conConf = con.getConnectionConfig();
        assert conConf != null;

        HttpClient httpClient;
        if (allowDeflate || allowGzip)
            httpClient = new DecompressingHttpClient(con.getHttpClient());
        else
            httpClient = con.getHttpClient();
        assert httpClient != null;


        Date startTime = new Date();
        HttpUriRequest httpCall = null;

        URI uri;
        try {
            URIBuilder uriBuilder = new URIBuilder(baseUrl);
            for (Map.Entry<String, Object> e : urlParameters.entrySet()) {
                uriBuilder.addParameter(e.getKey(), e.getValue() + "");
            }
            uri = uriBuilder.build();
        } catch (URISyntaxException e) {
            //we don't change the signature of this method, so we have to translate this error.
            throw new MalformedURLException(e.getMessage());
        }

        switch (httpVerb) {
            case GET: {
                httpCall = new HttpGet(uri);
                assert requestEntity == null : "Cannot send content with HTTP GET";
                break;
            }
            case POST: {
                HttpPost h = new HttpPost(uri);
                httpCall = h;
                if (requestEntity != null) {
                    h.setEntity(requestEntity);
                }
                break;
            }
            case PUT: {
                HttpPut h = new HttpPut(uri);
                httpCall = h;
                if (requestEntity != null) {
                    h.setEntity(requestEntity);
                }
                break;
            }
            case DELETE: {
                httpCall = new HttpDelete(uri);
                LOG.warn("Sending content with a HTTP DELETE call! This is rather unexpected..");
                break;
            }
        }
        assert httpCall != null;

        if (extraHeaders != null)
            for (Map.Entry<String, String> e : extraHeaders.entrySet())
                httpCall.addHeader(e.getKey(), e.getValue());

        httpCall.addHeader("Accept", "*/*");
        if (allowGzip || allowDeflate) {
            String allowed = "";
            if (allowGzip)
                allowed += "gzip";
            if (allowDeflate && allowGzip)
                allowed += ", ";
            if (allowDeflate)
                allowed += "deflate";

            httpCall.addHeader("Accept-Encoding", allowed);
        } else {
            //accept no encodings
        }

        StringBuilder httpRequestHeaders = new StringBuilder(200);
        for (Header header : httpCall.getAllHeaders())
            httpRequestHeaders.append(header.toString()).append("\n");
        httpRequestHeaders.append("\n");

        Date stopTime;
        int timeoutRetries = 0;
        boolean tryAgain = true;
        while (tryAgain) //repeat until first occurrence of a successful return or an error
            try {
                con.markInUse();
                tryAgain = false;
                LOG.trace("AbstractApi.executeHttpCall executing call to \"" + uri.toASCIIString() + "\"");
                startTime = new Date();
                stopTime = null;
                HttpResponse response =
                        con.getHttpContext() != null ?
                                httpClient.execute(httpCall, con.getHttpContext()) :
                                httpClient.execute(httpCall);
                stopTime = new Date();
                LOG.trace("AbstractApi.executeHttpCall call returned statusline \"" + response.getStatusLine() + "\"");

                boolean busy = false;
                if (response.getStatusLine() != null && response.getStatusLine().getStatusCode() == 500/*Busy*/)
                    busy = true;

                StringBuilder httpResponseHeaders = new StringBuilder(200);
                for (Header header : response.getAllHeaders())
                    httpResponseHeaders.append(header.toString()).append("\n");
                httpResponseHeaders.append("\n");

                HttpEntity entity = response.getEntity();

                String resultContent = entity != null ? IOUtils.streamToString(entity.getContent(), "UTF-8") : "";
                LOG.trace("AbstractApi.executeHttpCall call returned: \"" + resultContent + "\"");

                HttpCallDetails httpCallDetails = new DefaultHttpCallDetails(conConf, uri.toASCIIString(),
                        httpCall.getRequestLine().toString(), httpRequestHeaders.toString(), requestEntity,
                        response.getStatusLine(), httpResponseHeaders.toString(), resultContent,
                        startTime, stopTime, null/*exception*/, con);

                //no logging here
//            logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, null);
                return httpCallDetails;


//         } catch (ConnectTimeoutException | SocketTimeoutException e) {  //this would work in java 7
            } catch (InterruptedIOException e) {
                stopTime = new Date();
                LOG.debug("AbstractApi.executeHttpCall call threw InterruptedIOException (typically SocketTimeoutException or ConnectTimeoutException)", e);

                HttpCallDetails httpCallDetails = new DefaultHttpCallDetails(conConf, uri.toASCIIString(),
                        httpCall.getRequestLine().toString(), httpRequestHeaders.toString(), requestEntity,
                        null, null, null,
                        startTime, stopTime, e, con);

                if (timeoutRetries++ >= retrySettings.getTimeoutMaxRetries()) {
                    String extra = "";
                    if (timeoutRetries - 1 > 0)
                        extra = "Already retried " + timeoutRetries + " times after timeout. ";
                    throw new JFedTimeoutException("AbstractApi.executeHttpCall call threw timeout exception. " + extra + "Possibly incorrect URL \"" + con.getServerUrl() + "\". Could also be connectivity problem.",
                            e,
                            null,
                            GeniAMResponseCode.SERVER_TIMEOUT_ERROR);
                }

                //handle timeout: we will log and retry
                logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, e);

                LOG.info("Got read timeout (timeoutRetries=" + timeoutRetries + ") from server. Waiting " + retrySettings.getTimeoutMsBeforeRetry() + "ms, and trying again...");
                try {
                    Thread.sleep(retrySettings.getTimeoutMsBeforeRetry());
                } catch (InterruptedException ex) {
                    LOG.warn("InterruptedException in wait after timeout. Possible reason: User interrupted wait?", ex);
                    //user propably interrupted us.
                    //Note: this will probably result in double logging...
                    httpCall.abort();
                    throw new JFedException("Aborted waiting after timeout", ex, null, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                }
                tryAgain = true;
                httpCall.abort();
            } catch (javax.net.ssl.SSLPeerUnverifiedException ex) {
                HttpCallDetails httpCallDetails =
                        new DefaultHttpCallDetails(conConf, uri.toASCIIString(), httpCall.getRequestLine().toString(),
                                httpRequestHeaders.toString(), requestEntity, null, null, null, startTime, new Date(), ex, con);
                logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, ex);
                httpCall.abort();
                throw new JFedConnectivityException("The server certificate for '" + uri.toASCIIString() + "' could not be verified. " +
                        "Error message \"" + ex.getMessage() + "\". " +
                        "Possible causes:\n" +
                        "  - The server's self-signed certificate is not in our trust store. \n" +
                        "  - The server's certificate \"CN\" field is not the server hostname or a known alias. \n" +
                        "  - The server is not accepting our client login certificate+key pair (perhaps is is not federated with our \"login provider\"). \n" +
                        "  - If the server's certificate is not self signed, we might not have the root certificate of the trust chain in our trust store.", ex);
            } catch (SSLException ex) {
                HttpCallDetails httpCallDetails =
                        new DefaultHttpCallDetails(conConf, uri.toASCIIString(), httpCall.getRequestLine().toString(),
                                httpRequestHeaders.toString(), requestEntity, null, null, null, startTime, new Date(), ex, con);
                logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, ex);
                httpCall.abort();
                throw new JFedConnectivityException("SSLException: " + ex.getMessage(), ex);
            } catch (ClientProtocolException ex) {
                HttpCallDetails httpCallDetails =
                        new DefaultHttpCallDetails(conConf, uri.toASCIIString(), httpCall.getRequestLine().toString(),
                                httpRequestHeaders.toString(), requestEntity, null, null, null, startTime, new Date(), ex, con);
                logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, ex);
                httpCall.abort();
                throw new JFedConnectivityException("ClientProtocolException: " + ex.getMessage(), ex);
            } catch (IOException ex) {
                HttpCallDetails httpCallDetails =
                        new DefaultHttpCallDetails(conConf, uri.toASCIIString(), httpCall.getRequestLine().toString(),
                                httpRequestHeaders.toString(), requestEntity, null, null, null, startTime, new Date(), ex, con);
                logNonSfa(httpCallDetails, null, javaMethodName, apiMethodName, con, methodParameters, ex);
                httpCall.abort();
                throw new JFedConnectivityException("IOException: " + ex.getMessage(), ex);
            } finally {
                con.markNotInUse();
            }

        //this should never be reached!
        throw new RuntimeException("Bug in code, this should never occur.");
    }

            /*
            example of 504 seen:

            HTTP/1.1 504 Gateway Time-out

            <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
            <html><head>
            <title>504 Gateway Time-out</title>
            </head><body>
            <h1>Gateway Time-out</h1>
            <p>The gateway did not receive a timely response
            from the upstream server or application.</p>
            <hr>
            <address>Apache/2.2.22 (FreeBSD) Server at www.emulab.net Port 443</address>
            </body></html>

            * */


    protected XMLRPCCallDetailsWithCodeValueError executeXmlRpcCommandGeni(SfaConnection con, String command, List argv, Map<String, Object> methodParameters) throws JFedException {
        XMLRPCCallDetails res = executeXmlRpcCommand(con, command, argv, methodParameters, true);
        if (!(res instanceof XMLRPCCallDetailsWithCodeValueError)) {
            XMLRPCCallDetailsWithCodeValueError convertedres = new XMLRPCCallDetailsWithCodeValueError(
                    res.getConnectionConfig(), res.getServerUrl(),
                    res.getRequestHttpRequestLine(), res.getRequestHttpHeaders(), res.getRequestHttpContent(),
                    res.getResultHttpStatusLineObject(), res.getResultHttpHeaders(), res.getResultHttpContent(),
                    res.getRequest(),
                    (Map) res.getResult(), res.getStartTime(), res.getStopTime(), res.getException(), con
            );
            return convertedres;
        } else
            return (XMLRPCCallDetailsWithCodeValueError) res;
    }

    /**
     * @param xmlRpcClientReply the reply
     * @param con               the connection object
     * @param xMLRPCCallDetails the reply converted to XMLRPCCallDetails, or null if that is not possible
     * @returns true if the provided Object (returned by xmlRpcClient.execute) means the server is busy
     * false does not mean the server isn't busy, just that this hasn't been derived from the returned object.
     */
    protected boolean isBusy(Object xmlRpcClientReply, SfaConnection con, XMLRPCCallDetails xMLRPCCallDetails) {
        LOG.trace("isBusy called");

        if (con != null &&
                con.getXmlRpcTransportFactory() != null &&
                con.getXmlRpcTransportFactory().getHttpStatusLine() != null) {
            int code = con.getXmlRpcTransportFactory().getHttpStatusLine().getStatusCode();
            if (code == 503 /* 503 Service Unavailable */)
                return true;
        }

        if (xmlRpcClientReply == null) return false;

        if (xmlRpcClientReply instanceof XmlRpcException) {
            org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) xmlRpcClientReply;
            LOG.info("XmlRpc call returned XmlRpcException of class=" + ex.getClass().getName() + " code=" + ex.code + " msg=" + ex.getMessage(), ex);

            if (isBusy(ex)) return true;
        }
        LOG.trace("isBusy xMLRPCCallDetails=" + xMLRPCCallDetails + " xmlRpcClientReply=" + xmlRpcClientReply.getClass().getName());

        if (xMLRPCCallDetails != null/* && xmlRpcClientReply instanceof Map*/) {
            //check for API specific busy codes
            //this is used for example to check for geni AM return codes 11, 14 and -32001
            if (isBusyReply(xMLRPCCallDetails))
                return true;

//            //I doubt this is correct?  (there is no http return code 11, geni return code 11 does however mean busy, and that should be handled by isBusyReply)
//            if (xMLRPCCallDetails.getResultHttpStatusCode() == 11 /*unavailable, ProtoGeni SA returns this*/)
//                return true;
        }

        LOG.trace("not busy");

        return false;
    }

    /**
     * @returns true if the provided Object (returned by xmlRpcClient.execute) means the server has somehow timed out
     * false does not mean the server hasn't timed out, just that this hasn't been derived from the returned object.
     */
    protected boolean isTimeout(Object xmlRpcClientReply, SfaConnection con) {
        if (con != null &&
                con.getXmlRpcTransportFactory() != null &&
                con.getXmlRpcTransportFactory().getHttpStatusLine() != null) {
            int code = con.getXmlRpcTransportFactory().getHttpStatusLine().getStatusCode();
            if (code == 504 /* 504 Gateway Timeout */)
                return true;

            if (code == 408 /* 408 Request Timeout */)
                return true;
            if (code == 419 /* 419 Authentication Timeout  */)
                return true;
            if (code == 524 /* 524 A timeout occurred (Cloudflare) */)
                return true;
            if (code == 598 /* 598 Network read timeout error (Unknown) */)
                return true;
            if (code == 599 /* 599 Network connect timeout error (Unknown) */)
                return true;
        }

        if (xmlRpcClientReply == null) return false;

        if (xmlRpcClientReply.getClass().equals(org.apache.xmlrpc.XmlRpcException.class)) {
            org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) xmlRpcClientReply;
//            LOG.info("XmlRpc call returned XmlRpcException of class=" + ex.getClass().getName() + " code=" + ex.code + " msg=" + ex.getMessage(), ex);

            if (isTimeout(ex)) return true;
        }

        return false;
    }

    /**
     * @return true if the provided Object (returned by xmlRpcClient.execute) means the server has somehow timed out
     * false does not mean the server hasn't timed out, just that this hasn't been derived from the returned object.
     */
    protected boolean isPossibleTemporaryError(Object xmlRpcClientReply, SfaConnection con, XMLRPCCallDetails xMLRPCCallDetails) {
        if (xmlRpcClientReply == null) return false;

        if (xmlRpcClientReply.getClass().equals(org.apache.xmlrpc.XmlRpcException.class)) {
            org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) xmlRpcClientReply;
            if (isPossibleTemporaryError(ex)) return true;
        }

        if (xMLRPCCallDetails != null) {
            //check for API specific busy codes
            //this is used for example to check for geni AM return codes 11, 14 and -32001
            if (isPossibleTemporaryErrorReply(xMLRPCCallDetails))
                return true;
        }

        return false;
    }

    /**
     * @return true if the provided Throwable (thrown by xmlRpcClient.execute) means the server possibly had a temporary failure
     * false does not mean the server hasn't had a temporary failure, just that this hasn't been derived from the returned Exception.
     */
    protected boolean isPossibleTemporaryError(Throwable e) {
        if (e == null) return false;

        if (e.getClass().equals(org.apache.xmlrpc.XmlRpcException.class)) {
            org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) e;
            //hack to treat "Could not verify user certificate chain" as "possibly temporary error"
            if (ex.code == -32500 && ex.getMessage().equals("Could not verify user certificate chain"))
                return true;
        }

        if (isPossibleTemporaryErrorHelper(e)) return true;

        //loop deeper into causes
        while (e.getCause() != null) {
            e = e.getCause();
            if (isTimeoutHelper(e)) return true;
        }

        return false;
    }

    protected boolean isPossibleTemporaryErrorHelper(Throwable e) {
        if (e == null) return false;

        if (e instanceof javax.net.ssl.SSLPeerUnverifiedException) {
            if (e.getMessage().trim().contains("peer not authenticated"))
                return true;
        }

        return false;
    }

    /**
     * @returns true if the provided Exception (thrown by xmlRpcClient.execute) means the server is busy
     * false does not mean the server isn't busy, just that this hasn't been derived from the returned Exception.
     */
    protected boolean isBusy(Exception xmlRpcClientException) {
        if (xmlRpcClientException == null) return false;

        if (xmlRpcClientException.getClass().equals(org.apache.xmlrpc.XmlRpcException.class)) {
            org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) xmlRpcClientException;

            if (ex.code == 503) {
               /*example:   (previously checked this only for XmlRpcClientException)
                *    <methodResponse>
                *    <fault>
                *    <value><struct>
                *    <member><name>faultString</name><value><string>Server is WAY too busy; please try again later</string></value></member>
                *    <member><name>faultCode</name><value><i4>503</i4></value></member>
                *    </struct></value>
                *    </fault>
                *    </methodResponse>
                */
                return true;
            }

            LOG.debug("Checking exception for busy returned negative. code=" + ex.code + " msg=\"" + ex.getMessage() + "\"");

            if (ex instanceof XmlRpcClientException && xmlRpcClientException.getCause() != null) {
                //See if any cause of this exception, (no matter how deep), is a HttpServerBusyException. In that case, don't throw the error.
                Throwable recurseEx = ex;
                while (recurseEx != null) {
                    recurseEx = recurseEx.getCause();
                    if (recurseEx != null && recurseEx instanceof CommonsHttpClientXmlRpcTransportFactory.HttpServerErrorException) {
                        return true;
                    }
                }
            }
        }

        return false;
    }

    protected XMLRPCCallDetails executeXmlRpcCommand(SfaConnection con, String command, List argv, Map<String, Object> methodParameters, boolean geni) throws JFedException {
        Date startTime = null;
        Date stopTime = null;
        int busyRetries = 0;
        int possibleTempErrorRetries = 0;
        int timeoutRetries = 0;
        try {
            con.markInUse();
            boolean useNewConnection = false;
            while (true) { //loop is broken by return of an answer (optionally a non "Busy") or an exception.
                if (useNewConnection) {
                    if (!con.canDuplicate())
                        LOG.error("Should have retried with new connection, but the connection cannot create a new copy of itself. Will fall back to retry on same connection.");
                    else {
                        JFedConnection newCon = con.getNewConnection();
                        if (newCon != null)
                            con = (SfaConnection) newCon;
                        else
                            LOG.error("Should have retried with new connection, but the connection failed to create a new copy of itself. Will fall back to retry on same connection.");
                    }
                }
                useNewConnection = false; //false unless set again below

                XmlRpcClient xmlRpcClient = con.getXmlRpcClient();
                lastHttpResult = null;
                lastXmlRpcResult = null;

                startTime = new Date();
                con.markCallTime();
                stopTime = null;

                Object rc = null;
                Exception e = null;
                XMLRPCCallDetails res = null;
                try {
                    con.logWillExecuteXmlRpc(command, argv);
                    rc = xmlRpcClient.execute(command, argv);
                    stopTime = new Date();

                    //Check Keep-alive and make connection reuse adhere to its limits
                    //This is needed because emulab returns a "Could not verify user certificate chain" error if the
                    //   timeout is exceeded for some users (not for emulab users, even from other sites.
                    //    But for example geni portal users are affected)
                    //see FEDIBBTDEV-763
                    //  note that this is actually a workaround for a server bug (apache, mod_ssl or emulab bug or misconfiguration)
                    //  because the Keep-alive header is normally only informative. See http://www.w3.org/Protocols/HTTP/1.1/draft-ietf-http-v11-spec-01.html#Keep-Alive
                    //  a server should close the TCP connection after the timeout, instead of returning these strange unrelated exceptions.
                    if (con != null && con.getXmlRpcTransportFactory() != null && con.getXmlRpcTransportFactory().getHttpResponseHeaders() != null) {
                        List<Header> responseHeaders = con.getXmlRpcTransportFactory().getRawHttpResponseHeaders();

                        for (Header h : responseHeaders) {
                            if (h.getName().equalsIgnoreCase("Keep-Alive")) {
                                String val = h.getValue();

                                Matcher matcherTimeout = PATTERN_KEEP_ALIVE_TIMEOUT.matcher(val);
                                if (matcherTimeout.find()) {
                                    String timeoutS = matcherTimeout.group(1);
                                    try {
                                        long connectionExpireMs = Integer.parseInt(timeoutS) * 1000L;
                                        //respect the keep-alive
                                        con.reduceConnectionExpireMs(connectionExpireMs);
                                    } catch (NumberFormatException e1) {
                                        LOG.warn("Invalid timeout in HTTP header Keep-Alive: \"" + val + "\": \"" + timeoutS + "\"", e1);
                                    }
                                }
                                break;
                            }
                        }
                    }

                    final boolean useGeneralXmlRpcDetails;
                    if (rc == null) {
                        useGeneralXmlRpcDetails = true;
                    } else {
                        if (rc.getClass().equals(XmlRpcException.class)) {
                            useGeneralXmlRpcDetails = true;
                        } else {
                            if (geni) {
                                if (rc instanceof HashMap) {
                                    HashMap r = (HashMap) rc;
                                    useGeneralXmlRpcDetails = false;
                                    res = new XMLRPCCallDetailsWithCodeValueError(con.getConnectionConfig(),
                                            con.getServerUrl(),
                                            con.getXmlRpcTransportFactory().getHttpRequestLine(),
                                            con.getXmlRpcTransportFactory().getHttpRequestHeaders(),
                                            con.getXmlRpcTransportFactory().getHttpSentHistory(),
                                            con.getXmlRpcTransportFactory().getHttpStatusLine(),
                                            con.getXmlRpcTransportFactory().getHttpResponseHeaders(),
                                            con.getXmlRpcTransportFactory().getHttpReceivedHistory(),
                                            argv, r, startTime, stopTime, null, con);
                                } else
                                    throw new RuntimeException("Geni reply is not a Map! It is " + (rc == null
                                            ? "null" : "a" + rc.getClass().getName()));
                            } else {
                                useGeneralXmlRpcDetails = true;
                            }
                        }
                    }
                    if (useGeneralXmlRpcDetails)
                        res = new XMLRPCCallDetailsGeneral(con.getConnectionConfig(),
                                con.getServerUrl(),
                                con.getXmlRpcTransportFactory().getHttpRequestLine(),
                                con.getXmlRpcTransportFactory().getHttpRequestHeaders(),
                                con.getXmlRpcTransportFactory().getHttpSentHistory(),
                                con.getXmlRpcTransportFactory().getHttpStatusLine(),
                                con.getXmlRpcTransportFactory().getHttpResponseHeaders(),
                                con.getXmlRpcTransportFactory().getHttpReceivedHistory(),
                                argv, rc, startTime, stopTime, null, con);
                }


//                catch (InterruptedException ie) {
//                    would be nice if this handled interruptedExceptions
//                    these would be thrown when the task interrupts this thread when it is doing xmlRpcClient.execute
//                    however, it seems that execute does not handle interrupts and thus does not throw them...
//                }


//                //CODE NOT NEEDED: this exception is handled in isTimeout(e) and isBusy(e)
//                //catch (ConnectTimeoutException | SocketTimeoutException e) {  //this would work in java 7
//                catch (InterruptedIOException ex) {
//                    e = ex;
//                }
                catch (Exception ex) {
                    e = ex;
                    stopTime = new Date();
                    HttpInfoXmlRpcTransportFactory fact = con == null ? null : con.getXmlRpcTransportFactory();
                    res = new XMLRPCCallDetailsGeneral(con == null ? null : con.getConnectionConfig(),
                            con == null ? null : con.getServerUrl(),
                            fact == null ? null : fact.getHttpRequestLine(),
                            fact == null ? null : fact.getHttpRequestHeaders(),
                            fact == null ? null : fact.getHttpSentHistory(),
                            fact == null ? null : fact.getHttpStatusLine(),
                            fact == null ? null : fact.getHttpResponseHeaders(),
                            fact == null ? null : fact.getHttpReceivedHistory(),
                            argv, null, startTime, stopTime, e, con);
//                    log(res, null, null, command, con, methodParameters); no logging needed here
                }

                //whatever happened, res will have been created using all available details
                assert res != null;

                final boolean timeout = isTimeout(e) || isTimeout(rc, con);
                if (timeout) {
                    if (timeoutRetries++ >= retrySettings.getTimeoutMaxRetries()) {
                        String extra = "";
                        if (timeoutRetries - 1 > 0)
                            extra = "Already retried " + timeoutRetries + " times after timeout. ";
                        throw new JFedTimeoutException("XML RPC call for command \"" + command + "\" with argv size=" + argv.size() + " threw timeout exception. " + extra + "Possibly incorrect URL \"" + con.getServerUrl() + "\". Could also be connectivity problem.",
                                e,
                                res,
                                GeniAMResponseCode.SERVER_TIMEOUT_ERROR);
                    }

                    //handle timeout: we will log and retry
                    log(res, null, null, command, con, methodParameters);

                    LOG.info("Got read timeout (timeoutRetries=" + timeoutRetries + ") from server. Waiting " + retrySettings.getTimeoutMsBeforeRetry() + " milliseconds, and trying again...");
                    try {
                        Thread.sleep(retrySettings.getTimeoutMsBeforeRetry());
                    } catch (InterruptedException ex) {
                        LOG.warn("InterruptedException in wait after timeout. Possible reason: User interrupted wait?", ex);
                        //user propably interrupted us.
                        //Note: this will probably result in double logging...
                        throw new JFedException("Aborted waiting after timeout", ex, res, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                    }

                    //if timeout, always try again on a new connection
                    useNewConnection = true;

                    //reset possibleTempErrorRetries and busyRetries
                    possibleTempErrorRetries = 0;
                    busyRetries = 0;

                    continue; //try call again
                }

                final boolean busy = isBusy(e) || isBusy(rc, con, res);
                if (busy) {
                    if (busyRetries++ >= retrySettings.getBusyMaxRetries()) {
                        LOG.warn("Received \"Busy\" reply from server. NOT waiting and retrying: busyRetries=" + busyRetries +
                                " maxBusyCount=" + retrySettings.getBusyMaxRetries());
                        String extra = "";
                        if (busyRetries - 1 > 0)
                            extra =
                                    ", and already retried " + busyRetries + " times after busy (which is the maximum number)";
                        throw new JFedException("XML RPC call for command \"" + command + "\" with argv size=" + argv.size() + " received 'BUSY' reply" + extra + ".",
                                res,
                                GeniAMResponseCode.GENIRESPONSE_BUSY);
                    }

                    //log in case of busy
                    log(res, null, null, command, con, methodParameters);

                    LOG.info("Received \"Busy\" reply from server. Waiting " + retrySettings.getBusyMsBeforeRetry() + " milliseconds, and trying again..." +
                            " busyRetries=" + busyRetries + " maxBusyCount=" + retrySettings.getBusyMaxRetries());
                    try {
                        Thread.sleep(retrySettings.getBusyMsBeforeRetry());
                    } catch (InterruptedException ex) {
                        LOG.warn("InterruptedException in wait after busy. Possible reason: User interrupted wait?", e);
                        //user propably interrupted us.
                        //Note: this will probably result in double logging...
                        throw new JFedException("Aborted waiting after busy", ex, res, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                    }

                    //if busy, only try again on a new connection if must_reconnect_each_call is set.
                    //note: this uses DebugInfo, which is a bit of a hack. But it works if the info is available.
                    //Otherwise, we fall back to always use a new connection. (which is acceptable)
                    if (con.getDebugInfo() != null && con.getDebugInfo().getService() != null) {
                        Service service = con.getDebugInfo().getService();
                        useNewConnection = service.getServer().hasFlag(Server.Flag.workaroundMustReconnectEachCall);
                    } else
                        useNewConnection = true;

                    //reset possibleTempErrorRetries and timeoutRetries
                    possibleTempErrorRetries = 0;
                    timeoutRetries = 0;

                    continue; //try call again
                }

                final boolean possibleTempError = isPossibleTemporaryError(e) || isPossibleTemporaryError(rc, con, res);
                if (possibleTempError) {
                    if (possibleTempErrorRetries++ >= retrySettings.getPossibleTemporaryErrorMaxRetries()) {
                        LOG.warn("Received \"Possible Temporary error\" reply from server. NOT waiting and retrying: possibleTempErrorRetries=" + possibleTempErrorRetries +
                                " getPossibleTemporaryErrorMaxRetries()=" + retrySettings.getPossibleTemporaryErrorMaxRetries());
                        String extra = "";
                        if (possibleTempErrorRetries - 1 > 0)
                            extra =
                                    ", and already retried " + possibleTempErrorRetries + " times after possibleTempError (which is the maximum number)";
                        throw new JFedException("XML RPC call for command \"" + command + "\" with argv size=" + argv.size() + " received possible temporary error reply" + extra + ".",
                                res,
                                GeniAMResponseCode.GENIRESPONSE_BUSY);
                    }

                    //log in case of possibleTempError
                    log(res, null, null, command, con, methodParameters);

                    LOG.info("Received possible temporary error from server. Waiting " + retrySettings.getPossibleTemporaryErrorMsBeforeRetry() + " milliseconds, and trying again..." +
                            " possibleTempErrorRetries=" + possibleTempErrorRetries + " maxPossibleTemporaryErrorCount=" + retrySettings.getPossibleTemporaryErrorMaxRetries());
                    try {
                        Thread.sleep(retrySettings.getPossibleTemporaryErrorMsBeforeRetry());
                    } catch (InterruptedException ex) {
                        LOG.warn("InterruptedException in wait after possibleTempError. Possible reason: User interrupted wait?", e);
                        //user propably interrupted us.
                        //Note: this will probably result in double logging...
                        throw new JFedException("Aborted waiting after possibleTempError", ex, res, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                    }

                    //always use a new connection for this kind of error.
                    useNewConnection = true;

                    //reset timeoutRetries but don't reset busyRetries
                    timeoutRetries = 0;
                    //busyRetries = 0;

                    continue; //try call again
                }

                assert !timeout;
                assert !busy;
                assert !possibleTempError;

                boolean possiblyIncorrectURL = false;
                if (e != null) {
                    //TODO check for exception types that point to URL problem, instead of assuming all
                    boolean sawSslException = false;
                    boolean sawHostnameAndCertificateMismatchSSLException = false;
                    boolean sawXmlRpcException = false;
                    boolean sawSocketException = false;
                    boolean sawHttpHostConnectException = false;
                    Throwable curEx = e;
                    while (curEx != null) {
                        if (curEx instanceof XmlRpcException)
                            sawXmlRpcException = true;
                        if (curEx instanceof SocketException) //ConnectException is a SocketException
                            sawSocketException = true;
                        if (curEx instanceof HttpHostConnectException)
                            sawHttpHostConnectException = true;
                        if (curEx instanceof SSLException)
                            sawSslException = true;
                        if (curEx instanceof HostnameAndCertificateMismatchSSLException)
                            sawHostnameAndCertificateMismatchSSLException = true;

                        curEx = curEx.getCause();
                    }
                    possiblyIncorrectURL = sawSocketException || sawHttpHostConnectException || (!sawXmlRpcException && !sawSslException && !sawHostnameAndCertificateMismatchSSLException);

                    throw new JFedConnectivityException("XML RPC call for command \"" + command + "\" with argv size=" + argv.size() + " threw exception." +
                            "(Con.ID=" + con.getConnectionId() + ") " +
                            (possiblyIncorrectURL ? " Is URL correct? \"" + con.getServerUrl() + "\"" :
                                    (sawHostnameAndCertificateMismatchSSLException ?
                                            "Server certificate did not match the server hostname. " +
                                                    "This error is typically caused by a (minor) server SSL misconfiguration combined with java's paranoid security. " +
                                                    "jFed can be configured to make java trust the SSL certificate despite of this (but is not configured that way for this server yet)."
                                            : "")),
                            e,
                            res,
                            GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                }

                if (rc == null) {
                    possiblyIncorrectURL = true;
                    //This mostly occurs if the server returns an empty answer, and for emulab that can happen for some wrong URLs
                    throw new JFedConnectivityException("XML RPC call for command \"" + command + "\" with argv size=" + argv.size() + " returned null." +
                            (possiblyIncorrectURL ? " Is URL correct? \"" + con.getServerUrl() + "\"" : ""),
                            res,
                            GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                }

                if (rc.getClass().equals(org.apache.xmlrpc.XmlRpcException.class)) {
                    org.apache.xmlrpc.XmlRpcException ex = (org.apache.xmlrpc.XmlRpcException) rc;
                    LOG.info("XmlRpc call returned XmlRpcException of class=" + ex.getClass().getName() + " code=" + ex.code + " msg=" + ex.getMessage(), ex);

                    throw new JFedException("XML RPC call returned error: " + ex.getMessage(), ex, res, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);
                }
                if (geni && !(rc instanceof Map))
                    throw new JFedException("XMLRPC execute result class unexpected. This API is expected to " +
                            "return a Map, containing the entries code, value and output. Instead, it " +
                            "returned: " + rc.getClass().getName(), res, GeniAMResponseCode.INTERNAL_NONGENI_ERROR);

                //if we got here, we have an acceptable answer to the call, and will return it
                //  there is no need to log, the caller of this method has to do that
                //    (we can't because we can't create the APICallReply)
                lastHttpResult = res;
                lastXmlRpcResult = res;
                LOG.trace("AbstactAPI.executeXmlRpcCommand answer");
                return res;
            } //while
        } catch (final Exception e) {
            LOG.debug("AbstactAPI.executeXmlRpcCommand exception: ", e);
            XMLRPCCallDetails res;
            if (!(e instanceof JFedException) || ((JFedException) e).getXmlRpcResult() == null) {
                if (con.getXmlRpcTransportFactory() != null)
                    res = new XMLRPCCallDetailsGeneral(con.getConnectionConfig(),
                            con.getServerUrl(),
                            con.getXmlRpcTransportFactory().getHttpRequestLine(),
                            con.getXmlRpcTransportFactory().getHttpRequestHeaders(),
                            con.getXmlRpcTransportFactory().getHttpSentHistory(),
                            con.getXmlRpcTransportFactory().getHttpStatusLine(),
                            con.getXmlRpcTransportFactory().getHttpResponseHeaders(),
                            con.getXmlRpcTransportFactory().getHttpReceivedHistory(),
                            argv, null, startTime, stopTime, e, con);
                else
                    res = new XMLRPCCallDetailsGeneral(con.getConnectionConfig(),
                            con.getServerUrl(),
                            null, null, null, null, null, null,
                            argv, null, startTime, stopTime, e, con);
            } else
                res = ((JFedException) e).getXmlRpcResult();

            lastHttpResult = res;
            lastXmlRpcResult = res;
            log(res, new ApiCallReply<String>() {
                @Override
                public GeniAMResponseCode getGeniResponseCode() {
                    return GeniAMResponseCode.INTERNAL_NONGENI_ERROR;
                }

                @Override
                public String getValue() {
                    return e.getMessage();
                }

                @Override
                public String getOutput() {
                    return "CLIENT SIDE ERROR while executing command: " + e.getMessage() + "\n";
                }

                @Override
                public Map getRawResult() {
                    return null;
                }

                @Override
                public Object getRawValue() {
                    return getValue();
                }
            }, null, command, con, methodParameters, e);
            con.markError();
            throw new JFedException("Exception in XML RPC call: " + e.getMessage(), e, res, GeniAMResponseCode.SERVER_REPLY_ERROR);
        } finally {
            con.markNotInUse();

            //remove data about last call stored in connection object
            con.clearLastCallData();
        }
    }

    public enum HttpVerb {GET, POST /*create*/, PUT /*update*/, DELETE}

    public static class DefaultHttpCallDetails implements HttpCallDetails {
        private final ConnectionConfig connectionConfig;
        private final String serverUrl;
        private final String requestHttpRequestLine;
        private final String requestHttpHeaders;
        private final String requestHttpContent;
        private final StatusLine resultHttpStatusLine;
        private final String resultHttpHeaders;
        private final String resultHttpContent;
        private final Date startTime;
        private final Date stopTime;
        private final Throwable exception;
        private final JFedConnection connection;

        public DefaultHttpCallDetails(ConnectionConfig connectionConfig, String serverUrl,
                                      String requestHttpRequestLine, String requestHttpHeaders, HttpEntity requestHttpEntity,
                                      StatusLine resultHttpStatusLine, String resultHttpHeaders, String resultHttpContent,
                                      Date startTime, Date stopTime, Throwable exception, JFedConnection connection) {
            this.connectionConfig = connectionConfig;
            this.serverUrl = serverUrl;
            this.requestHttpRequestLine = requestHttpRequestLine;
            this.requestHttpHeaders = requestHttpHeaders;
            this.resultHttpStatusLine = resultHttpStatusLine;
            this.resultHttpHeaders = resultHttpHeaders;
            this.resultHttpContent = resultHttpContent;
            this.startTime = startTime;
            this.stopTime = stopTime;
            this.exception = exception;
            this.connection = connection;

            if (requestHttpEntity != null) {
                String requestHttpContentHelper;
                try {
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    requestHttpEntity.writeTo(baos);
                    baos.close();
                    requestHttpContentHelper = baos.toString("UTF-8");
                } catch (IOException e) {
                    LOG.warn("Error writing requestHttpEntity for logging.", e);
                    requestHttpContentHelper = null;
                }
                this.requestHttpContent = requestHttpContentHelper;
            } else
                this.requestHttpContent = null;
        }

        @Override
        public String getServerUrl() {
            return serverUrl;
        }

        @Override
        public ConnectionConfig getConnectionConfig() {
            return connectionConfig;
        }

        @Override
        public String getRequestHttpContent() {
            return requestHttpContent;
        }

        @Override
        public int getResultHttpStatusCode() {
            return resultHttpStatusLine == null ? -1 : resultHttpStatusLine.getStatusCode();
        }

        @Override
        public String getResultHttpStatusReasonPhrase() {
            return resultHttpStatusLine == null ? null : resultHttpStatusLine.getReasonPhrase();
        }

        @Override
        public String getRequestHttpHeaders() {
            return requestHttpHeaders;
        }

        @Override
        public String getRequestHttpRequestLine() {
            return requestHttpRequestLine;
        }

        @Override
        public String getResultHttpHeaders() {
            return resultHttpHeaders;
        }

        @Override
        public String getResultHttpStatusLine() {
            return resultHttpStatusLine == null ? null : resultHttpStatusLine.toString();
        }

        @Override
        public StatusLine getResultHttpStatusLineObject() {
            return resultHttpStatusLine;
        }

        @Override
        public String getResultHttpContent() {
            return resultHttpContent;
        }

        @Override
        public Date getStartTime() {
            return startTime;
        }

        @Override
        public Date getStopTime() {
            return stopTime;
        }

        @Override
        public Throwable getException() {
            return exception;
        }

        @Override
        public JFedConnection getLastConnection() {
            return connection;
        }
    }

    /**
     * XMLRPCCallDetails contains all info about an XmlRpc call on XmlRpc level and below.
     * This means it offers access to the actual sent/received HTTP and XmlRpc data.
     * It also offers access to interpreted results and request of the call (as List and Map)
     * <p/>
     * XMLRPCCallDetailsWithCodeValueError assumes the result is a Map with "code" "value" and "error" fields.
     * It does not assume anything about the type of "code"
     */
    public static class XMLRPCCallDetailsWithCodeValueError implements be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails {
        private final String resultXmlRpcString;
        private final String requestXmlRpcString;
        private final List request;
        private final Map result;
        private final String serverUrl;
        private final ConnectionConfig connectionConfig;
        private final Date startTime;
        private final Date stopTime;
        private final Throwable exception;
        private final String requestHttpRequestLine;
        private final String requestHttpHeaders;
        private final String requestHttpContent;
        private final StatusLine resultHttpStatusLine;
        private final String resultHttpHeaders;
        private final String resultHttpContent;
        private final JFedConnection connection;

        /**
         * @param connectionConfig
         * @param requestHttpContent
         * @param resultHttpContent
         * @param request
         * @param result             may be null
         */
        public XMLRPCCallDetailsWithCodeValueError(ConnectionConfig connectionConfig,
                                                   String serverUrl,
                                                   String requestHttpRequestLine, String requestHttpHeaders, String requestHttpContent,
                                                   StatusLine resultHttpStatusLine, String resultHttpHeaders, String resultHttpContent,
                                                   List request,
                                                   Map result, Date startTime, Date stopTime, Throwable exception, JFedConnection connection) {
            this.requestHttpRequestLine = requestHttpRequestLine;
            this.requestHttpHeaders = requestHttpHeaders;
            this.requestHttpContent = requestHttpContent;
            this.resultHttpStatusLine = resultHttpStatusLine;
            this.resultHttpHeaders = resultHttpHeaders;
            this.resultHttpContent = resultHttpContent;
            if (result != null)
                this.resultXmlRpcString = XmlRpcPrintUtil.xmlRpcObjectToString(result);
            else
                this.resultXmlRpcString = "No result";
            this.requestXmlRpcString = XmlRpcPrintUtil.xmlRpcObjectToString(request);
            this.request = request;
            this.result = result;
            this.connectionConfig = connectionConfig;
            this.serverUrl = serverUrl;

            this.startTime = startTime;
            this.stopTime = stopTime;

            this.connection = connection;
            this.exception = exception;
        }

        public String getServerUrl() {
            return serverUrl;
        }

        @Override
        public ConnectionConfig getConnectionConfig() {
            return connectionConfig;
        }

        @Override
        public String getRequestHttpContent() {
            return requestHttpContent;
        }

        @Override
        public String getRequestHttpHeaders() {
            return requestHttpHeaders;
        }

        @Override
        public int getResultHttpStatusCode() {
            return resultHttpStatusLine.getStatusCode();
        }

        @Override
        public String getResultHttpStatusReasonPhrase() {
            return resultHttpStatusLine.getReasonPhrase();
        }

        @Override
        public String getResultHttpStatusLine() {
            return resultHttpStatusLine.toString();
        }

        @Override
        public StatusLine getResultHttpStatusLineObject() {
            return resultHttpStatusLine;
        }

        @Override
        public String getRequestHttpRequestLine() {
            return requestHttpRequestLine;
        }

        @Override
        public String getResultHttpHeaders() {
            return resultHttpHeaders;
        }

        @Override
        public String getResultHttpContent() {
            return resultHttpContent;
        }

        @Override
        public String getResultXmlRpcString() {
            return resultXmlRpcString;
        }

        @Override
        public String getRequestXmlRpcString() {
            return requestXmlRpcString;
        }

        public List getRequest() {
            return request;
        }

        public Map getResult() {
            return result;
        }

        @Override
        public Throwable getException() {
            return exception;
        }

        @Override
        public JFedConnection getLastConnection() {
            return connection;
        }

        @Override
        public String getResultValueString() {
            if (result == null) return "";
            Object value = result.get("value");
            if (value == null) return "";
            return value.toString();
        }

        @Override
        public Object getResultValueObject() {
            if (result == null) return null;
            Object value = result.get("value");
            return value;
        }

        //        @Override
        public Object getResultCode() {
            if (result == null) return null;
            return result.get("code");
        }

        //        @Override
        public String getResultOutput() {
            if (result == null) return "";
            String output = ((String) result.get("output"));
            return output;
        }


        public Date getStartTime() {
            return startTime;
        }

        public Date getStopTime() {
            return stopTime;
        }
    }

    /**
     * XMLRPCCallDetails contains all info about an XmlRpc call on XmlRpc level and below.
     * This means it offers access to the actual sent/received HTTP and XmlRpc data.
     * It also offers access to interpreted results and request of the call (as List and Map)
     */
    public static class XMLRPCCallDetailsGeneral implements be.iminds.ilabt.jfed.lowlevel.XMLRPCCallDetails {
        private final String resultXmlRpcString;
        private final String requestXmlRpcString;
        private final List request;
        private final Object result;
        private final String serverUrl;
        private final ConnectionConfig connectionConfig;
        private final Date startTime;
        private final Date stopTime;
        private final Throwable exception;
        private final String requestHttpRequestLine;
        private final String requestHttpHeaders;
        private final String requestHttpContent;
        private final StatusLine resultHttpStatusLine;
        private final String resultHttpHeaders;
        private final String resultHttpContent;
        private final JFedConnection connection;

        /**
         * @param connectionConfig
         * @param requestHttpContent
         * @param resultHttpContent
         * @param request
         * @param result             may be null
         */
        public XMLRPCCallDetailsGeneral(ConnectionConfig connectionConfig,
                                        String serverUrl,
                                        String requestHttpRequestLine, String requestHttpHeaders, String requestHttpContent,
                                        StatusLine resultHttpStatusLine, String resultHttpHeaders, String resultHttpContent,
                                        List request, Object result, Date startTime, Date stopTime, Throwable exception,
                                        JFedConnection connection) {
            this.requestHttpRequestLine = requestHttpRequestLine;
            this.requestHttpHeaders = requestHttpHeaders;
            this.requestHttpContent = requestHttpContent;
            this.resultHttpStatusLine = resultHttpStatusLine;
            this.resultHttpHeaders = resultHttpHeaders;
            this.resultHttpContent = resultHttpContent;
            if (result != null)
                this.resultXmlRpcString = XmlRpcPrintUtil.xmlRpcObjectToString(result);
            else
                this.resultXmlRpcString = "No result";
            this.requestXmlRpcString = XmlRpcPrintUtil.xmlRpcObjectToString(request);
            this.request = request;
            this.result = result;
            this.serverUrl = serverUrl;
            this.connectionConfig = connectionConfig;

            this.startTime = startTime;
            this.stopTime = stopTime;

            this.exception = exception;
            this.connection = connection;
        }

        @Override
        public String getServerUrl() {
            return serverUrl;
        }

        @Override
        public ConnectionConfig getConnectionConfig() {
            return connectionConfig;
        }

        @Override
        public String getRequestHttpRequestLine() {
            return requestHttpRequestLine;
        }

        @Override
        public String getResultHttpHeaders() {
            return resultHttpHeaders;
        }

        @Override
        public String getRequestHttpContent() {
            return requestHttpContent;
        }

        @Override
        public int getResultHttpStatusCode() {
            return resultHttpStatusLine.getStatusCode();
        }

        @Override
        public String getResultHttpStatusReasonPhrase() {
            return resultHttpStatusLine.getReasonPhrase();
        }

        @Override
        public StatusLine getResultHttpStatusLineObject() {
            return resultHttpStatusLine;
        }

        @Override
        public String getRequestHttpHeaders() {
            return requestHttpHeaders;
        }

        @Override
        public String getResultHttpStatusLine() {
            if (resultHttpStatusLine == null) return null;
            return resultHttpStatusLine.toString();
        }

        @Override
        public String getResultHttpContent() {
            return resultHttpContent;
        }

        @Override
        public String getResultXmlRpcString() {
            return resultXmlRpcString;
        }

        @Override
        public String getRequestXmlRpcString() {
            return requestXmlRpcString;
        }

        @Override
        public List getRequest() {
            return request;
        }

        @Override
        public Object getResult() {
            return result;
        }

        @Override
        public Throwable getException() {
            return exception;
        }

        @Override
        public JFedConnection getLastConnection() {
            return connection;
        }

        @Override
        public String getResultValueString() {
            return result.toString();
        }

        @Override
        public Object getResultValueObject() {
            return result;
        }

        public Date getStartTime() {
            return startTime;
        }

        public Date getStopTime() {
            return stopTime;
        }
    }

    protected static class CompressedStringConverter implements ReplyConverter<String> {
        private final Boolean compressed;

        public CompressedStringConverter(Boolean compressed) {
            this.compressed = compressed;
        }

        @Override
        public String convertXMLRPCCall(Object resultValueObject) throws BadReplyGeniException {
            if (resultValueObject instanceof String) {
                String resultValueString = (String) resultValueObject;
                if (resultValueString.trim().isEmpty())
                    return null;

                if (Objects.equals(Boolean.TRUE, compressed)) {
                    return DataConversionUtils.decompressFromBase64(resultValueString);
                } else {
                    return resultValueString;
                }
            } else
                return null;
        }
    }
}
