package be.iminds.ilabt.jfed.lowlevel.connection;

import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Server;
import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Service;
import be.iminds.ilabt.jfed.lowlevel.ApiInfo;
import be.iminds.ilabt.jfed.lowlevel.GeniUser;
import be.iminds.ilabt.jfed.lowlevel.JFedException;
import be.iminds.ilabt.jfed.lowlevel.api.ProtoGeniAMExtensions;
import be.iminds.ilabt.jfed.preferences.CorePreferenceKey;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.util.JFedPasswordManager;
import be.iminds.ilabt.jfed.util.JFedTrustStore;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;

/**
 * SfaConnectionPool
 * <p/>
 * A connection pool used for reusing connections. Users of this class can request connections, and the SfaConnectionPool
 * will either create them, or return an already existing connection.
 * <p/>
 * Connections with errors are automatically removed.
 * <p/>
 * Only connections with the same settings will be selected for reuse. See {@link be.iminds.ilabt.jfed.lowlevel.connection.SfaConnectionPool.ConnectionSpecification} for details.
 * This includes these settings: authority, api, disableClientAuth, proxyInfo
 * <p/>
 * The SfaConnectionPool also stores a bae trust store, to use for all new connections.
 * When connections to an authority are requested, a trust store to which the certificates of that authority are added will be used.
 * When connections without any specified authority are used, the caller can specify a trust store to use (or null to use the SfaConnectionPool trust store).
 * <p/>
 * The SfaConnectionPool stores default proxy info to use. Some calls allow this default to be overwritten.
 */
@Singleton
public class SfaConnectionPool implements JFedConnectionProvider {
    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(SfaConnectionPool.class);
    private final Map<ConnectionSpecification, JFedConnection> conPool
            = Collections.synchronizedMap(new HashMap<ConnectionSpecification, JFedConnection>());
    private final JFedTrustStore baseTrustStore;
    private final JFedPreferences jFedPreferences;
    private final ConnectionBuilderFactory connectionBuilderFactory;
    private final JFedPasswordManager passwordManager;
    private final long firstDate = System.currentTimeMillis();
    private JFedConnection.ProxyInfo defaultProxy = null;
    private long lastThreadCpuDump = 0;
    private final Map<Long, Long> prev_thread_cpuNs_sinceStart_ByThreadId = new HashMap<>();
    private long prevCpuNsSinceStart = 0;
    private long prevDate = System.currentTimeMillis();
    private boolean debugCpuUsage = false;



    @Inject
    public SfaConnectionPool(JFedTrustStore baseTrustStore, final JFedPreferences jFedPreferences,
                             ConnectionBuilderFactory connectionBuilderFactory, JFedPasswordManager passwordManager) {
        this.baseTrustStore = baseTrustStore;
        this.jFedPreferences = jFedPreferences;
        this.connectionBuilderFactory = connectionBuilderFactory;
        this.passwordManager = passwordManager;
        startAutoExpire();

        if (jFedPreferences.hasKey(CorePreferenceKey.PREF_DEBUG_CPU)) {
            setDebugCpuUsage(jFedPreferences.getBoolean(CorePreferenceKey.PREF_DEBUG_CPU));
            jFedPreferences.addPreferenceChangeListener(CorePreferenceKey.PREF_DEBUG_CPU,
                    () -> setDebugCpuUsage(jFedPreferences.getBoolean(CorePreferenceKey.PREF_DEBUG_CPU)));
        }
    }

    public int size() {
        synchronized (conPool) {
            return conPool.size();
        }
    }

    public JFedConnection.ProxyInfo getDefaultProxy() {
        return defaultProxy;
    }

    public void setDefaultProxy(JFedConnection.ProxyInfo defaultProxy) {
        this.defaultProxy = defaultProxy;
    }

    /**
     * Forget all cached connections, so that existing connections cannot be reused and new connections will be made.
     */
    public void flush() {
        //we don't want to close active connections, because they can still be in use in a call

        synchronized (conPool) {
            conPool.clear();
        }
    }

    private JFedPasswordManager.LoginInfo getSimpleLogin(ApiInfo.ApiName api, GeniUser user) {
        final String serviceName;
        final String serviceFriendlyName;
        switch (api) {
            case GTS_API: {
                serviceName = "gts";
                serviceFriendlyName = "GTS";
                break;
            }
            case FED4FIRE_SLA_COLLECTOR: {
                serviceName = "sla";
                serviceFriendlyName = "SLA Collector";
                break;
            }
            default:
                throw new RuntimeException("Login retrieval not implemented for service " + api);
        }
        List<JFedPasswordManager.LoginInfo> logins =
                passwordManager.getLoginsAndRequestIfNone(serviceName, serviceFriendlyName, jFedPreferences, user);
        if (logins.isEmpty())
            return null;
        return logins.get(0);
    }

    @Override
    public JFedConnection getConnectionByAuthority(@Nullable GeniUser user, @Nonnull Server server,
                                                   @Nonnull Class targetClass) throws JFedException {
        return getConnectionByAuthority(user, server, targetClass, false);
    }

    public JFedConnection getConnectionByAuthority(@Nullable GeniUser user, @Nonnull Server server,
                                                   @Nonnull Class targetClass, boolean disableClientAuthentication)
            throws JFedException {

        ApiInfo.Api api = ApiInfo.classToApi(targetClass);
        if (api != null)
            return getConnectionByAuthority(user, server, api, disableClientAuthentication);
        if (Objects.equals(targetClass, ProtoGeniAMExtensions.class)) {
            //AMv2 OR AMv3
            JFedConnection con =
                    getConnectionByAuthority(user, server, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2), disableClientAuthentication);
            if (con != null) return con;
            return getConnectionByAuthority(user, server, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3), disableClientAuthentication);
        }
        throw new JFedException("Cannot get connection for " + targetClass.getName());
    }

    public JFedConnection getConnectionByUserAuthority(@Nonnull GeniUser user, @Nonnull ApiInfo.Api api)
            throws JFedException {
        return getConnectionByUserAuthority(user, api, false);
    }

    public JFedConnection getConnectionByUserAuthority(@Nonnull GeniUser user, @Nonnull ApiInfo.Api api,
                                                       boolean disableClientAuthentication) throws JFedException {
        assert user.getUserAuthorityServer() != null :
                "getConnectionByUserAuthority requires that the user's authority is known";
        Server server = user.getUserAuthorityServer();
        assert server != null;

        return getConnectionByAuthority(user, server, api, disableClientAuthentication);
    }

    /* TODO: connections by URL are not cached yet*/
    @Override
    public JFedConnection getConnectionByUrl(JFedTrustStore overrideTrustStore, GeniUser user, URL serverUrl,
                                             HandleUntrustedCallback handleUntrustedCallback, ApiInfo.Api api,
                                             Service serviceForDebugInfo, List<String> overrideAllowedServerCertificateAlias) throws JFedException {
        return getConnectionByUrl(overrideTrustStore, user, serverUrl, handleUntrustedCallback, api,
                false/*disableClientAuthentication*/, false/*overrideProxy*/, null, serviceForDebugInfo, overrideAllowedServerCertificateAlias);
    }

    public JFedConnection getConnectionByUrl(JFedTrustStore overrideTrustStore, GeniUser user, URL serverUrl,
                                             HandleUntrustedCallback handleUntrustedCallback, ApiInfo.Api api,
                                             boolean disableClientAuthentication, Service serviceForDebugInfo, List<String> overrideAllowedServerCertificateAlias)
            throws JFedException {
        return getConnectionByUrl(overrideTrustStore, user, serverUrl, handleUntrustedCallback, api,
                disableClientAuthentication, false/*overrideProxy*/, null, serviceForDebugInfo, overrideAllowedServerCertificateAlias);
    }

    public JFedConnection getConnectionByUrl(JFedTrustStore overrideTrustStore, GeniUser user, URL serverUrl,
                                             HandleUntrustedCallback handleUntrustedCallback, ApiInfo.Api api,
                                             boolean disableClientAuthentication, boolean overrideProxy,
                                             JFedConnection.ProxyInfo overrideProxyInfo, Service serviceForDebugInfo, List<String> overrideAllowedServerCertificateAlias)
            throws JFedException {
        assert api != null;

        final JFedTrustStore conTrustStore = overrideTrustStore == null ? baseTrustStore : overrideTrustStore;

        JFedConnection.ProxyInfo usedProxyInfo;
        if (overrideProxy)
            usedProxyInfo = overrideProxyInfo;
        else
            usedProxyInfo = defaultProxy;

        JFedConnection con;

        if (serverUrl == null)
            throw new RuntimeException("getConnectionByUrl: serverUrl null for api: " + api);

        ConnectionBuilder connectionBuilder = connectionBuilderFactory.createConnectionBuilder();
        connectionBuilder.setUrl(serverUrl);
        connectionBuilder.setDebugInfo(new JFedConnection.DebugInfo(serviceForDebugInfo, user));
        connectionBuilder.setProxy(usedProxyInfo, false);
        connectionBuilder.setHackOverrideAllowedServerCertificateAlias(overrideAllowedServerCertificateAlias);

        switch (api.getName().getAuthentication()) {
            case NONE: {
                connectionBuilder.useNoAuthentication();
                break;
            }
            case HTTP_BASIC: {
                final JFedPasswordManager.LoginInfo login = getSimpleLogin(api.getName(), user);
                if (login == null) {
                    LOG.error("Failed to get login info, will not create connection.");
                    return null;
                }

                connectionBuilder.useHttpBasicAuthentication(login);
                break;
            }
            case SSL_CLIENT_AUTH: {
                assert user != null : "HTTPS connections require that a user is logged in";
                connectionBuilder.useSslClientAuthentication(user.getClientCertificateChain(), user.getPrivateKey());
                break;
            }
            default:
                throw new RuntimeException("Unhandled Authentication: " + api.getName().getAuthentication());
        }

        if (Objects.equals(serverUrl.getProtocol(), "https"))
            connectionBuilder.useHttps(conTrustStore, handleUntrustedCallback);
        else {
            connectionBuilder.useHttp();

            if (api.getName().getAuthentication() == ConnectionConfig.Authentication.SSL_CLIENT_AUTH) {
                LOG.warn("This API (" + api + ") normally uses SSL CLIENT AUTHENTICATION. However, the server URL uses HTTP instead of HTTPS. " +
                        "Because of this, SSL client authentication cannot be used and will be DISABLED.");
                connectionBuilder.useNoAuthentication();
            }
        }

        switch (api.getName().getConnectionType()) {
            case HTTP: {
                con = connectionBuilder.buildHttpConnection();
                break;
            }
            case SFA: {
                con = connectionBuilder.buildSfaConnection();
                break;
            }
            default:
                throw new RuntimeException("Unhandled Connection Type: " + api.getName().getConnectionType());
        }

        assert con != null;
        return con;
    }

    @Override
    public JFedConnection getConnectionByAuthority(GeniUser user, @Nonnull Server server, @Nonnull ApiInfo.Api api) throws JFedException {
        return getConnectionByAuthority(user, server, api, false, false, null);
    }

    public JFedConnection getConnectionByAuthority(GeniUser user, Server server, ApiInfo.Api api,
                                                   boolean disableClientAuthentication) throws JFedException {
        return getConnectionByAuthority(user, server, api, disableClientAuthentication, false, null);
    }

    public JFedConnection getConnectionByAuthority(GeniUser user, Server server, ApiInfo.Api api,
                                                   boolean disableClientAuthentication,
                                                   JFedConnection.ProxyInfo overwriteProxyInfo) throws JFedException {
        return getConnectionByAuthority(user, server, api, disableClientAuthentication, true, overwriteProxyInfo);
    }

    /**
     * @param overrideProxy      if true, use parameter overwriteProxyInfo as proxyInfo (even if null!). If false use proxy settings in SfaConnectionPool (and overwriteProxyInfo is ignored)
     * @param overwriteProxyInfo the proxy info to overwrite the default SfaConnectionPool proxy info with. Note that this can be null while overrideProxy=true, to disable proxy.
     */
    public JFedConnection getConnectionByAuthority(GeniUser user, @Nonnull Server server, @Nonnull ApiInfo.Api api,
                                                   boolean disableClientAuthentication,
                                                   boolean overrideProxy, JFedConnection.ProxyInfo overwriteProxyInfo)
            throws JFedException {
        JFedConnection.ProxyInfo usedProxyInfo = overrideProxy ? overwriteProxyInfo : defaultProxy;

        if (server.getServices() == null || server.getServices().isEmpty())
            throw new JFedException("Server has no services: " + "server ID: " + server.getId());

        Service usedService = null;
        for (Service service : server.getServices()) {
            if (api.matches(service))
                usedService = service;
        }

        if (usedService == null)
            throw new JFedException("Server has no services for api '" + api.getId() + "' server ID: " + server.getId());

        ConnectionSpecification conSpec =
                new ConnectionSpecification(user, server, usedService, disableClientAuthentication, usedProxyInfo);
        return getConnectionByConnectionSpecification(conSpec);
    }

    public JFedConnection getConnectionByConnectionSpecification(@Nonnull ConnectionSpecification connectionSpecification) throws JFedException {
        GeniUser user = connectionSpecification.getUser();
        Server server = connectionSpecification.getServer();
        Service service = connectionSpecification.getService();
        boolean disableClientAuthentication = connectionSpecification.isDisableClientAuth();

        assert server != null;
        assert service != null;

        JFedConnection con;
        synchronized (conPool) {
            con = conPool.get(connectionSpecification);
        }

        if (con == null)
            LOG.trace("No cached connection for " + connectionSpecification);
        if (con != null && (con.isError() || con.isExpired())) {
            //something went wrong, we'll kill the connection
            LOG.debug("NOT reusing cached connection {} for {} due to error in connection.", con.getConnectionId(), connectionSpecification);

            synchronized (conPool) {
                conPool.remove(connectionSpecification);
            }
            con.close();
            con = null;
        }
        if (con != null)
            LOG.trace("Re-using cached connection {} for {}", con.getConnectionId(), connectionSpecification);

        if (con == null) {
            JFedConnection.JFedConnectionFactory conCopier;
            URL serverUrl;
            try {
                serverUrl = new URL(service.getUrl());
            } catch (MalformedURLException e) {
                throw new JFedException("Service (id=" + service.getId() + ") has invalid URL: \"" + service.getUrl() + "\"", e);
            }

            final JFedTrustStore conTrustStore = new JFedTrustStore(baseTrustStore);

            final ConnectionBuilder connectionBuilder = connectionBuilderFactory.createConnectionBuilder();
            connectionBuilder.setUrl(serverUrl);
            connectionBuilder.setDebugInfo(new JFedConnection.DebugInfo(connectionSpecification.getService(), user));
            connectionBuilder.setProxy(connectionSpecification.getProxyInfo(), false);

            ApiInfo.ApiName api = ApiInfo.findById(service.getApi());
            if (api == null)
                throw new JFedException("API is now known: "
                        + "api: \"" + service.getApi() + "\"   "
                        + "service ID: " + service.getId() + "   "
                        + "server ID: " + server.getId());


            switch (api.getAuthentication()) {
                case HTTP_BASIC: {
                    final JFedPasswordManager.LoginInfo login = getSimpleLogin(api, user);
                    if (login == null) {
                        LOG.error("Failed to get login info, will not create connection.");
                        return null;
                    }

                    connectionBuilder.useHttpBasicAuthentication(login);
                    break;
                }
                case SSL_CLIENT_AUTH: {
                    if (!disableClientAuthentication) {
                        assert user != null : "HTTPS connections require that a user is logged in";
                        connectionBuilder.useSslClientAuthentication(user.getClientCertificateChain(), user.getPrivateKey());
                        break;
                    }
                    //else fall-through
                }
                case NONE: {
                    connectionBuilder.useNoAuthentication();
                    break;
                }
                default:
                    throw new RuntimeException("Unhandled Authentication: " + api.getAuthentication());
            }

            if (Objects.equals(serverUrl.getProtocol(), "https")) {
                if (server.hasCertificateChain()) {
                    conTrustStore.addTrustedPemCertificateIfNotAdded(server.getCertificateChain());
                    if (server.hasAllowedCertificateAlias()) {
                        conTrustStore.addAllowedServerCertificateHostnameAlias(server.getAllowedCertificateAlias());
                        LOG.debug("Using SSL conTrustStore with cert and with alias \""+server.getAllowedCertificateAlias()+"\" " +
                                "for server " + server.getName() + " (" + server.getId()+")");
                    } else {
                        LOG.debug("Using SSL conTrustStore with cert but without alias for server "+server.getName()+" ("+server.getId()+")");
                    }
                } else {
                    LOG.debug("Using SSL conTrustStore without cert or alias info for server "+server.getName()+" ("+server.getId()+")");
                }
                if (user.getUserAuthorityServer() != null) {
                    if (user.getUserAuthorityServer().hasCertificateChain()) {
                        conTrustStore.addTrustedPemCertificateIfNotAdded(user.getUserAuthorityServer().getCertificateChain());
                        if (user.getUserAuthorityServer().hasAllowedCertificateAlias())
                            conTrustStore.addAllowedServerCertificateHostnameAlias(user.getUserAuthorityServer().getAllowedCertificateAlias());
                    }
                }
                connectionBuilder.useHttps(conTrustStore, null/*handleUntrustedCallback*/);
            } else {
                connectionBuilder.useHttp();

                if (api.getAuthentication() == ConnectionConfig.Authentication.SSL_CLIENT_AUTH) {
                    LOG.warn("This API (" + api + ") normally uses SSL CLIENT AUTHENTICATION. However, the server URL uses HTTP instead of HTTPS. " +
                            "Because of this, SSL client authentication cannot be used and will be DISABLED.");
                    connectionBuilder.useNoAuthentication();
                }
            }

            switch (api.getConnectionType()) {
                case HTTP: {
                    conCopier = new JFedConnection.JFedConnectionFactory() {
                        @Override
                        public JFedConnection create() throws JFedException {
                            JFedConnection res = connectionBuilder.buildHttpConnection();
                            assert res != null;
                            res.setConnectionFactory(this);
                            return res;
                        }
                    };
                    break;
                }
                case SFA: {
                    conCopier = new JFedConnection.JFedConnectionFactory() {
                        @Override
                        public JFedConnection create() throws JFedException {
                            SfaConnection res = connectionBuilder.buildSfaConnection();
                            assert res != null;
                            res.setConnectionFactory(this);
                            return res;
                        }
                    };
                    break;
                }
                default:
                    throw new RuntimeException("Unhandled Connection Type: " + api.getConnectionType());
            }

            con = conCopier.create(); //use copier to get initial connection itself
            LOG.trace("Created a new connection {} for {}", con.getConnectionId(), connectionSpecification);
            con.setConnectionFactory(conCopier);
            assert con.canDuplicate();
            assert con.getConnectionFactory() != null;

            //reuse connection next time, if allowed
            if (!server.hasFlag(Server.Flag.workaroundMustReconnectEachCall))
                synchronized (conPool) {
                    conPool.put(connectionSpecification, con);
                }
        }

        return con;
    }

    public void clear() {
        synchronized (conPool) {
            for (JFedConnection conn : conPool.values()) {
                LOG.debug("Closing connection to {}", conn.getServerUrl());
                conn.close();
            }

            conPool.clear();
        }
        LOG.debug("Cleared connection pool");
    }

    public void clearExpired() {
        int connectionsCleared = 0;

        //TODO: perhaps not a good idea to lock conPool while doing this. (though this should be a fast operation)
        synchronized (conPool) {

            for (Iterator<Map.Entry<ConnectionSpecification, JFedConnection>> it =
                 conPool.entrySet().iterator(); it.hasNext(); ) {
                Map.Entry<ConnectionSpecification, JFedConnection> entry = it.next();
                JFedConnection con = entry.getValue();
                if (con.isExpired()) {
                    LOG.debug("Closing expired connection to {}. inUse={} lastUse={} ms ago",
                            con.getServerUrl(),
                            con.isInUse(),
                            (System.currentTimeMillis() - con.getLastCallTime().getTime()));
                    con.close();
                    it.remove();
                    connectionsCleared++;
                }
            }
        }

        int conCount = conPool.size();

        if (connectionsCleared > 0)
            LOG.debug("Periodical check of expired connections in connection pool: Closed " + connectionsCleared + " (" + conCount + " remain open)");
        else
            LOG.debug("Periodical check of expired connections in connection pool: Did not find any. (" + conCount + " remain open)");

        if (debugCpuUsage)
            dumpCpuUsage();
    }

    public void setDebugCpuUsage(boolean debugCpuUsage) {
        this.debugCpuUsage = debugCpuUsage;
    }

    public void dumpCpuUsage() {
        ThreadMXBean tmxb = ManagementFactory.getThreadMXBean();
        if (!tmxb.isThreadCpuTimeSupported())
            LOG.debug("Thread CPU measurements are not supported");
        else {
            boolean dumpThreadCpuNow = (System.currentTimeMillis() - lastThreadCpuDump) > 60000; //once each minute

            StringBuilder logString = new StringBuilder();
            StringBuilder highCpulogString = new StringBuilder();

            long[] threadIDs = tmxb.getAllThreadIds();
            long cpuNsSinceStart = 0;
            for (long threadID : threadIDs) {
                long cpuTime = tmxb.getThreadCpuTime(threadID);
                cpuNsSinceStart += cpuTime;
            }
            long curDate = System.currentTimeMillis();
            long realMsSincePrev = curDate - prevDate;
            long realMsSinceStart = curDate - firstDate;
            long cpuNsSincePrev = cpuNsSinceStart - prevCpuNsSinceStart;

            double cpuPercentOfReal_sinceStart = (cpuNsSinceStart * 100.0) / (realMsSinceStart * 1000000.0);
            double cpuPercentOfReal_sincePrev = (cpuNsSincePrev * 100.0) / (realMsSincePrev * 1000000.0);

            for (long threadID : threadIDs) {
                ThreadInfo threadInfo = tmxb.getThreadInfo(threadID);
                if (threadInfo != null) {
                    String threadName = threadInfo.getThreadName();
                    long thread_cpuNs_sinceStart = tmxb.getThreadCpuTime(threadID); //in nanoseconds

                    Long prev_thread_cpuNs_sinceStart =
                            prev_thread_cpuNs_sinceStart_ByThreadId.get(threadID); //in nanoseconds
                    if (prev_thread_cpuNs_sinceStart == null) prev_thread_cpuNs_sinceStart = 0L;

                    long thread_cpuNs_sincePrev = thread_cpuNs_sinceStart - prev_thread_cpuNs_sinceStart;

                    double thread_cpuPercentOfProcess_sinceStart = (thread_cpuNs_sinceStart * 100.0) / cpuNsSinceStart;
                    double thread_cpuPercentOfProcess_sincePrev = (thread_cpuNs_sincePrev * 100.0) / cpuNsSincePrev;

                    double thread_cpuPercentOfReal_sinceStart =
                            (thread_cpuNs_sinceStart * 100.0) / (realMsSinceStart * 1000000.0);
                    double thread_cpuPercentOfReal_sincePrev =
                            (thread_cpuNs_sincePrev * 100.0) / (realMsSincePrev * 1000000.0);


                    logString.append("Thread ").append(threadID).append(" \"").append(threadName).append("\"\n")
                            .append("    from start:  cpuNs=").append(thread_cpuNs_sinceStart).append(" ")
                            .append(" (").append(thread_cpuPercentOfProcess_sinceStart).append("% of process cputime)")
                            .append(" (").append(thread_cpuPercentOfReal_sinceStart).append("% of real time)\n")
                            .append("    from prev:  cpuNS=").append(thread_cpuNs_sincePrev).append(" ")
                            .append(" (").append(thread_cpuPercentOfProcess_sincePrev).append("% of process cputime)")
                            .append(" (").append(thread_cpuPercentOfReal_sincePrev).append("% of real time)\n");

                    if (thread_cpuPercentOfProcess_sinceStart > 20 || thread_cpuPercentOfProcess_sincePrev > 20) {
                        highCpulogString.append("NOTE: HIGH CPU USE Thread ").append(threadID).append(" \"").append(threadName).append("\"").append("\n")
                                .append(" from start:  cpuNs=").append(thread_cpuNs_sinceStart).append(" ")
                                .append(" (").append(thread_cpuPercentOfProcess_sinceStart)
                                .append("% of process cputime)").append(" (").append(thread_cpuPercentOfReal_sinceStart)
                                .append("% of real time)").append("\n")
                                .append("  from prev:  cpuNS=").append(thread_cpuNs_sincePrev).append(" ")
                                .append(" (").append(thread_cpuPercentOfProcess_sincePrev)
                                .append("% of process cputime)")
                                .append(" (").append(thread_cpuPercentOfReal_sincePrev)
                                .append("% of real time)").append("\n")
                                .append("       Running for ").append(realMsSinceStart).append(" ms.     Used ")
                                .append(cpuNsSinceStart).append(" ns of cpu time in total. =").append(cpuPercentOfReal_sinceStart).append("%").append("\n")
                                .append("Last CPU print was ").append(realMsSincePrev).append(" ms ago. Used ").append(cpuNsSincePrev).append(" ns of cpu time in that time. =").append(cpuPercentOfReal_sincePrev).append("%\n");
                    }

                    prev_thread_cpuNs_sinceStart_ByThreadId.put(threadID, thread_cpuNs_sinceStart);
                }
            }
            logString.append("       Running for ").append(realMsSinceStart).append(" ms.     Used ").append(cpuNsSinceStart).append(" ns of cpu time in total. =").append(cpuPercentOfReal_sinceStart).append("%\n");
            logString.append("Last CPU print was ").append(realMsSincePrev).append(" ms ago. Used ").append(cpuNsSincePrev).append(" ns of cpu time in that time. =").append(cpuPercentOfReal_sincePrev).append("%");
            prevCpuNsSinceStart = cpuNsSinceStart;
            prevDate = curDate;

            if (dumpThreadCpuNow)
                LOG.info("Thread overview:\n" + logString.toString());
            else {
                if (LOG.isDebugEnabled())
                    LOG.debug("Thread overview:\n" + logString.toString());
                else
                    LOG.warn(highCpulogString.toString());
            }

            if (dumpThreadCpuNow)
                lastThreadCpuDump = System.currentTimeMillis();
        }
    }

    protected void startAutoExpire() {
        TimerTask expireConnectionsTask = new TimerTask() {
            @Override
            public void run() {
                clearExpired();
            }
        };

        //execute clearExpired each 10 seconds
        Timer timer = new Timer("SfaConnectionPool-AutoExpire", true /*is daemon*/);
        timer.schedule(
                expireConnectionsTask,
                10 * 1000L/*first delay (ms)*/,
                10 * 1000L /*period (ms)*/);
    }

    private static class ConnectionSpecification {
        private final GeniUser user;
        private final Server server;
        private final Service service;
        private final boolean disableClientAuth;
        private final JFedConnection.ProxyInfo proxyInfo;

        private ConnectionSpecification(GeniUser user, Server server, Service service, boolean disableClientAuth, JFedConnection.ProxyInfo proxyInfo) {
            if (server == null) throw new RuntimeException("server == null");
            if (service == null) throw new RuntimeException("service == null");
            this.user = user;
            this.service = service;
            this.server = server;
            this.disableClientAuth = disableClientAuth;
            this.proxyInfo = proxyInfo; //may be null
        }

        public Service getService() {
            return service;
        }

        public Server getServer() {
            return server;
        }

        public boolean isDisableClientAuth() {
            return disableClientAuth;
        }

        public JFedConnection.ProxyInfo getProxyInfo() {
            return proxyInfo;
        }

        public GeniUser getUser() {
            return user;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            ConnectionSpecification that = (ConnectionSpecification) o;

            if ((user == null) != (that.user == null)) return false;
            if (user != null && !Objects.equals(user.getUserUrn(), that.user.getUserUrn()))
                return false;
            if (!Objects.equals(service.getId(), that.service.getId())) return false;
            if (!Objects.equals(server.getId(), that.server.getId())) return false;
            if ((proxyInfo == null) != (that.proxyInfo == null)) return false;
            if (proxyInfo != null && !Objects.equals(proxyInfo, that.proxyInfo)) return false;
            if (disableClientAuth != that.disableClientAuth) return false;

            return true;
        }

        @Override
        public int hashCode() {
            int result = service.getId().hashCode();
            if (proxyInfo != null) result = 31 * result + proxyInfo.hashCode();
            result = 31 * result + server.getId().hashCode();
            result = 31 * result + (disableClientAuth ? 1 : 0);
            return result;
        }

        @Override
        public String toString() {
            return "ConnectionSpecification{" +
                    "user=" + (user == null ? "null" : user.getUserUrnString()) +
                    ", service=" + service.getId() +
                    ", server=" + server.getId() +
                    ", disableClientAuth=" + disableClientAuth +
                    ", defaultProxy=" + proxyInfo +
                    '}';
        }
    }
}
