package be.iminds.ilabt.jfed.scanner;

import be.iminds.ilabt.jfed.fedmon.webapi.service.json.*;
import be.iminds.ilabt.jfed.log.Logger;
import be.iminds.ilabt.jfed.lowlevel.*;
import be.iminds.ilabt.jfed.lowlevel.api.AbstractGeniAggregateManager;
import be.iminds.ilabt.jfed.lowlevel.api.AggregateManager2;
import be.iminds.ilabt.jfed.lowlevel.api.AggregateManager3;
import be.iminds.ilabt.jfed.lowlevel.api_wrapper.UserAndSliceApiWrapper;
import be.iminds.ilabt.jfed.lowlevel.api_wrapper.impl.AutomaticUserAndSliceApiWrapper;
import be.iminds.ilabt.jfed.lowlevel.connection.BasicConnectionBuilderFactory;
import be.iminds.ilabt.jfed.lowlevel.connection.ConnectionBuilder;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnection;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnectionProvider;
import be.iminds.ilabt.jfed.lowlevel.ssh_key_info.SshKeyInfoFactory;
import be.iminds.ilabt.jfed.lowlevel.stitching.VlanRange;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.rspec.model.BasicStringRspec;
import be.iminds.ilabt.jfed.util.*;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.stream.Collectors;

/**
 * ServerScanner: scans for all info about an authority, based on input data.
 */
public class ServerScanner {
    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ServerScanner.class);
    private static final Set<ServerPort> failedServerPorts = new HashSet<>();
    private final Logger logger;
    private final JFedPreferences jFedPreferences;
    private final AuthorityScannerInput input;
    private final ScanFeedback feedback;
    private final KeyStore trustedRootCerts;     //global trusted root certs
    private final Set<X509Certificate> trustedCerts;     //all certs, including input
    private final Set<X509Certificate> trustedAuthCerts; //all certs used by servers of this authority
    private String trustedAlias;
    public final Map<ApiInfo.Api, ScannedInformation<String>> urlsByApi =
            new HashMap<>();
    protected final List<AuthorityScannerLogListener> resultListeners = new ArrayList<>();
    private final ScannedInformation<AuthorityType> types = new ScannedInformation<>();
    private final List<String> logs = new ArrayList<>();
    private final ScannedInformation<String> hrns = new ScannedInformation<>();
    private final ScannedInformation<String> urns = new ScannedInformation<>();
    private final Set<ServerBase> serverBases = new HashSet<>();

    private Mode mode;

    private boolean connectionSuccess = false;
    private boolean getVersionSuccess = false;

    private final boolean useProxy;
    private final GeniUserProvider geniUserProvider;
    private final JFedConnectionProvider connectionProvider;
    private final GeniUser loggedInUser;
    private final BasicConnectionBuilderFactory connectionBuilderFactory = new BasicConnectionBuilderFactory();


    public ServerScanner(Logger logger,
                         AuthorityScannerInput input,
                         ScanFeedback feedback,
                         JFedPreferences jFedPreferences,
                         boolean useProxy,
                         @Nullable GeniUserProvider geniUserProvider,
                         JFedConnectionProvider connectionProvider) {
        this.logger = logger;
        this.jFedPreferences = jFedPreferences;
        this.input = input;
        this.feedback = feedback;
        this.useProxy = useProxy;
        this.connectionProvider = connectionProvider;
        this.geniUserProvider = geniUserProvider;
        this.loggedInUser = geniUserProvider == null ? null : geniUserProvider.getLoggedInGeniUser();
        trustedRootCerts = JFedTrustStore.getSystemTrustStore();
        trustedCerts = new HashSet<>(input.trustedCerts);
        trustedAuthCerts = new HashSet<>();
        trustedAlias = null;
    }

    public static Testbed createResult(ScannedInformation<AuthorityType> types,
                                       Map<ApiInfo.Api, ScannedInformation<String>> urlsByApi,
                                       ScannedInformation<String> hrns,
                                       ScannedInformation<String> urns,
                                       Set<X509Certificate> trustedAuthCerts,
                                       String trustedAlias) {
        ServerBuilder resultServer = new ServerBuilder();
        TestbedBuilder resultTestbed = new TestbedBuilder();
        resultTestbed.addServerBuilder(resultServer);

        String urnString = urns.getMostTrusted();
        GeniUrn urn;
        try {
            urn = new GeniUrn(urnString);
        } catch (GeniUrn.GeniUrnParseException e) {
            throw new RuntimeException("invalid URN found: \"" + urnString + "\"");
        }
        resultServer.setDefaultComponentManagerUrn(urnString);
        resultTestbed.setDefaultComponentManagerUrn(urnString);

        String testbedId = "CHANGE ME";
        testbedId = urn.getEncodedTopLevelAuthority_withoutSubAuth().replaceAll("[^A-Za-z0-9]", "");
        resultTestbed.setId(testbedId);

        String hrn = hrns.getMostTrusted();
        if (hrn == null)
            hrn = testbedId;
        if (hrn == null)
            hrn = "CHANGE ME";
        resultTestbed.setLongName(hrn);
        resultServer.setName(hrn);

        String usedURL = null;
        for (Map.Entry<ApiInfo.Api, ScannedInformation<String>> e : urlsByApi.entrySet()) {
            ApiInfo.Api api = e.getKey();
            ScannedInformation<String> urlInfo = e.getValue();

            if (!urlInfo.isEmpty()) {
                ServiceBuilder serviceBuilder = new ServiceBuilder();
                resultServer.addServiceBuilder(serviceBuilder);

                serviceBuilder.setApi(api.getId());
                serviceBuilder.setApiVersion(api.getVersion());
                serviceBuilder.setUrl(urlInfo.getMostTrusted());
                serviceBuilder.setUrn(adaptCmUrnToRole(api.getName(), urn));

                if (serviceBuilder.getUrl() != null)
                    usedURL = serviceBuilder.getUrl();
            }
        }

        if (usedURL != null) {
            try {
                URL url = new URL(usedURL);
                String baseUrl = usedURL.substring(0, usedURL.length() - url.getPath().length());
                resultServer.setBaseUrl(baseUrl);
            } catch (MalformedURLException e) {
                LOG.error("Error in used URL", e);
            }
        }

        if (urnString != null) {
            try {
                GeniUrn geniUrn = new GeniUrn(urnString);
                resultServer.setUrnTld(geniUrn.getEncodedTopLevelAuthority_withoutSubAuth());
            } catch (GeniUrn.GeniUrnParseException e) {
                LOG.error("Error in used URN", e);
            }
        }

        if (types.getMostTrusted() != null)
            resultServer.setServerType(types.getMostTrusted().toString());

        if (trustedAuthCerts != null && !trustedAuthCerts.isEmpty())
            resultServer.setCertificateChain(KeyUtil.x509certificateChainToPem(trustedAuthCerts));

        if (trustedAlias != null)
            resultServer.setAllowedCertificateAlias(trustedAlias);

        return resultTestbed.create();
    }


    private static String adaptCmUrnToRole(ApiInfo.ApiName apiName, GeniUrn urn) {
        String urnResourceName;
        switch (apiName) {
            case PLANETLAB_SLICE_REGISTRY: {
                urnResourceName = "sr";
                break;
            }
            case PROTOGENI_SA: {
                urnResourceName = "sa";
                break;
            }
            case GENI_AM: {
                if (Objects.equals(urn.getResourceName(), "am"))
                    urnResourceName = "am";
                else
                    urnResourceName = "cm";
                break;
            }
            case PROTOGENI_CH: {
                urnResourceName = "ch";
                break;
            }
            case GENI_CH: {
                urnResourceName = "ch";
                break;
            }
            case GENI_CH_SA: {
                urnResourceName = "sa";
                break;
            }
            case GENI_CH_MA: {
                urnResourceName = "ma";
                break;
            }
            case GENI_SCS: {
                urnResourceName = "scs";
                break;
            }
            case GTS_API:
            case FED4FIRE_FEDMON:
            case FED4FIRE_RESERVATION_CALENDER_PAGE:
            case FED4FIRE_SLA_COLLECTOR:
            case FED4FIRE_REPUTATION_SERVICE:
                return null;
            default:
                throw new RuntimeException("Unsupported api " + apiName);
        }
        return GeniUrn.createGeniUrnFromEncodedParts(urn.getEncodedTopLevelAuthority(), "authority", urnResourceName).toString();
    }


    public void addToUrlsByServerType(ScanCertainty certainty, ApiInfo.Api api, String url) {
        if (urlsByApi.containsKey(api)) {
            ScannedInformation<String> set = urlsByApi.get(api);
            set.add(certainty, url);
        } else {
            ScannedInformation<String> set = new ScannedInformation<>();
            set.add(certainty, url);
            urlsByApi.put(api, set);
        }
    }

    public Logger getLogger() {
        return logger;
    }

    private void log(String line) {
        logs.add(line);
        LOG.debug(line);
        fireLogLine(line);
    }

    private void log(String line, Throwable t) {
        String exceptionString = IOUtils.exceptionToStacktraceString(t);
        //String exceptionString = "  ERROR "+t.getClass().getSimpleName()+": "+t.getMessage();//TextUtil.exceptionToString(t);

        logs.add(line);
        if (t != null)
            logs.add(exceptionString);
        LOG.debug(line, t);
        fireLogLine(line + "\n" + exceptionString + "\n");
    }

    public void findServerBases() {
        for (String urn : input.urns) {
            GeniUrn u = GeniUrn.parse(urn);
            if (u != null)
                serverBases.add(new ServerBase(u.getEncodedTopLevelAuthority_withoutSubAuth(), 0, null));
        }
        for (String urlStr : input.urls) {
            try {
                URL url = new URL(urlStr);
                int port = url.getPort();
                if (port <= 0) port = url.getDefaultPort();
                String path = url.getPath();
                if (path.endsWith("/")) path = path.substring(0, path.length() - 1);
                if (path.startsWith("/")) path = path.substring(1);
                serverBases.add(new ServerBase(url.getHost(), port, path));
            } catch (MalformedURLException e) {
                log("Invalid URL: " + urlStr);
            }
        }
    }

    public void processInputUrl(String url) {
        //TODO put inputURL in toTestUrlsByServerType somehow
        addToUrlsByServerType(ScanCertainty.USER_INPUT, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2), url);
    }

    public void addUrlsForType(AuthorityType authType) {
        addUrlsForType(authType, ScanCertainty.GUESS);
    }

    public void addUrlsForType(AuthorityType authType, ScanCertainty scanCertainty) {
        for (ServerBase serverBase : serverBases)
            switch (authType) {
                case EMULAB: {
                    ServerBase knownPort = new ServerBase(serverBase.hostname, 12369, serverBase.path);
                    List<ServerBase> bases = new ArrayList<>();
                    bases.add(serverBase);
                    if (!Objects.equals(serverBase, knownPort))
                        bases.add(knownPort);
                    for (ServerBase base : bases) {
                        String serverBaseStr =
                                base.toString().replaceAll("^www.", "").replaceAll("protogeni/xmlrpc.*", "");
                        addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.PROTOGENI_SA, 2), "https://www." + serverBaseStr + "protogeni/xmlrpc/sa");
                        addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_CH_SA, 1), "https://www." + serverBaseStr + "protogeni/xmlrpc/geni-sa");
                        addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_CH_MA, 1), "https://www." + serverBaseStr + "protogeni/xmlrpc/geni-ma");
                        addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2), "https://www." + serverBaseStr + "protogeni/xmlrpc/am/2.0");
                        addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3), "https://www." + serverBaseStr + "protogeni/xmlrpc/am/3.0");
                    }
                    break;
                }
                case SFAWRAP: {
                    addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3),
                            "https://" + new ServerBase(serverBase.hostname, 12346, serverBase.path) + "");

                    addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.PLANETLAB_SLICE_REGISTRY, 1),
                            "https://" + new ServerBase(serverBase.hostname, 12345, serverBase.path) + "");
                    break;
                }
                case FOAM: {
                    String serverPath = serverBase.toString().replaceAll("foam/gapi.*", "");
                    addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 1),
                            "https://" + new ServerBase(serverBase.hostname, 3626, serverPath) + "foam/gapi/1");
                    addToUrlsByServerType(scanCertainty, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2),
                            "https://" + new ServerBase(serverBase.hostname, 3626, serverPath) + "foam/gapi/2");
                    break;
                }
                case GCF: {
                    //no specific URLs preconfigured in GCF
                    break;
                }
            }
    }


    private JFedConnection.SshProxyInfo getProxyInfo() {
        if (!useProxy) {
            return null;
        }
        if (loggedInUser == null) {
            log("No user logged in, so cannot use use proxy!");
            return null;
        }

        assert loggedInUser.getUserAuthorityServer() != null;
        List<Proxy> userProxies = loggedInUser.getUserAuthorityServer().getTestbed().getProxies();
        Optional<JFedConnection.SshProxyInfo> proxyInfo = userProxies.stream().findFirst().map(userProxy ->
                new JFedConnection.SshProxyInfo(userProxy.getHostname(), new VlanRange(userProxy.getPortRange()).getFirst(),
                        loggedInUser.getUserUrn().getEncodedResourceName(),
                        SshKeyInfoFactory.createGeniUserSshKeyInfo(loggedInUser),
                        userProxy.getHostKey(), true/*limitName*/));

        if (proxyInfo.isPresent()) {
            log("Selected proxy to use: " + proxyInfo.get());
            return proxyInfo.get();
        } else {
            log("No user proxy known, so cannot use use proxy!");
            return null;
        }
    }

    /**
     * Check if SSL connection can be set up securely. Does not call anything yet.
     */

    public boolean checkUrl(String urlStr) {
        URL url;
        try {
            url = new URL(urlStr);
            SSLCertificateDownloader.SSLCertificateJFedInfo info =
                    SSLCertificateDownloader.getCertificateInfo(url, getProxyInfo());
            assert info != null;

            if (info.isConnectionError()) {
                log("Note: Connection failure getting certificate info for \"" + urlStr + "\"", info.getConnectionException());
                return false;
            } else
                log("Note: got certificate info for \"" + urlStr + "\". " + info);

            connectionSuccess = info.getCert() != null;

            String urn;
            if (info.getUrn() == null) {
//                log("The URN of the server could not be determined from the certificate. Using placeholder URN for now.");
                urn = "urn:publicid:IDN+" + url.getHost() + "+authority+cm";
                urns.add(ScanCertainty.GUESS, urn);
            } else {
                urns.add(ScanCertainty.INDIRECT_RECV_FROM_SERVER, info.getUrn());
            }

            if (info.getCert() == null) {
                log("Problem fetching certificate from server for URL: " + urlStr);
                return false;
            }

            if (info.isSelfSigned() == null) {
                log("Problem fetching certificate from server for URL (not known if cert is self signed): " + urlStr);
                return false;
            }

            if (info.isSelfSigned() != null) {
                if (info.isSelfSigned()) {
                    if (!trustedCerts.contains(info.getCert())) {
                        log("Note: server uses an unknown self signed certificate:\n" +
                                "Issuer=" + info.getCert().getIssuerDN().getName() + "\n" +
                                "Subject=" + info.getCert().getSubjectDN().getName() + "\n" +
                                KeyUtil.x509certificateToPem(info.getCert()));

                        List<X509Certificate> certChain = new ArrayList<>();
                        certChain.add(info.getCert());
                        if (!feedback.trustSelfSignedCert(certChain)) {
                            log("user does not trust certificate for: " + urlStr);
                            return false;
                        } else {
                            trustedCerts.addAll(certChain);
                            trustedAuthCerts.addAll(certChain);
                            log("User trusts self signed certificate for " + urlStr);
                        }
                    } else {
                        log("Note: server uses a self signed certificate that is already trusted by jFed:\n" +
                                KeyUtil.x509certificateToPem(info.getCert()));
                    }
                } else {
                    assert !info.isSelfSigned();


                    //remove this globallyTrustedRootTest after debugging
                    boolean globallyTrustedRootTest;
                    try {
                        if (info.getChainRoot() == null)
                            globallyTrustedRootTest = false;
                        else {
                            String alias = trustedRootCerts.getCertificateAlias(info.getChainRoot());
                            globallyTrustedRootTest = alias != null;
                            LOG.debug("globallyTrustedRoot alias=" + alias);
                        }
                    } catch (KeyStoreException e) {
                        LOG.error("KeyStoreException while trying to check if certificate is globally trusted root -> Will assume it is not.", e);
                        globallyTrustedRootTest = false;
                    }
                    assert !globallyTrustedRootTest;


                    if (!trustedCerts.contains(info.getCert()) ||
                            (info.getChainRoot() != null && !trustedCerts.contains(info.getChainRoot()))) {
                        log("Note: server uses a non root, non self signed certificate:\n" +
                                "Issuer=" + info.getCert().getIssuerDN().getName() + "\n" +
                                "Subject=" + info.getCert().getSubjectDN().getName() + "\n" +
                                KeyUtil.x509certificateToPem(info.getCert()));

                        List<X509Certificate> certChain = new ArrayList<>();
                        certChain.add(info.getCert());
                        if (!feedback.trustNonSelfSignedCert(certChain)) {
                            log("user does not trust non self signed certificate for: " + urlStr);
                            return false;
                        } else {
                            trustedCerts.addAll(certChain);
                            trustedAuthCerts.addAll(certChain);
                            log("User trusts non-self signed certificate for " + urlStr);
                        }
                    } else {
                        boolean globallyTrustedRoot;
                        try {
                            globallyTrustedRoot =
                                    info.getChainRoot() != null && trustedRootCerts.getCertificateAlias(info.getChainRoot()) != null;
                        } catch (KeyStoreException e) {
                            LOG.error("KeyStoreException while trying to check if certificate is globally trusted root -> Will assume it is not.", e);
                            globallyTrustedRoot = false;
                        }

                        if (globallyTrustedRoot) {
                            log("Note: server uses a non self signed certificate signed by a globally trusted root certificate:\n" +
                                    info.getChainRoot().getIssuerDN().getName());
                            List<X509Certificate> certChain = new ArrayList<>();
                            certChain.add(info.getCert());
                            trustedCerts.addAll(certChain);
                            trustedAuthCerts.addAll(certChain);
                        } else {
                            log("Note: server uses a non root, non self signed certificate that is already trusted by jFed:\n" +
                                    KeyUtil.x509certificateToPem(info.getCert()));
                        }
                    }
                }
            }

            if (!info.getSubjectMatchesHostname() && url.getHost() != null && !Objects.equals(url.getHost(), info.getSubject())) {
                log("Note: Server certificate subject name is not for \"" + url.getHost() + "\" but for \"" + info.getSubject() + "\"");

                if (!feedback.trustAlias(url.getHost(), info.getSubject())) {
                    log("user does not trust certificate alias for: " + urlStr);
                    return false;
                } else {
                    trustedAlias = info.getSubject();
                    log("User trusts alias \"" + info.getSubject() + "\" in certificate.");
                }
            } else {
                log("Note: Server certificate subject name for server \"" + url.getHost() + "\" is \"" + info.getSubject() + "\"");
            }

            return true;
        } catch (MalformedURLException e) {
            log("Invalid URL: " + urlStr);
            return false;
        }
    }

    public String fixUrl(URL serverUrl, String getVersionUrl) {
        String serverBase = serverUrl.getHost();
        assert !serverBase.contains(":");
        if (serverUrl.getPort() >= 0)
            serverBase += ":" + serverUrl.getPort();

        String res = getVersionUrl;
        if (serverUrl.toString().startsWith("https:") && res.startsWith("http:"))
            res = res.replace("http:", "https:");

        if (res.contains(":" + serverUrl.getPort())) {
            res = res.replace("localhost:" + serverUrl.getPort(), serverBase);
            res = res.replace("127.0.0.1:" + serverUrl.getPort(), serverBase);
        } else {
            res = res.replace("localhost", serverBase);
            res = res.replace("127.0.0.1", serverBase);
        }
        return res;
    }

    /**
     * returns true if call successful
     */
    public boolean callAMGetVersion(@Nullable ApiInfo.Api api, URL url) {
        log("calling AM GetVersion on: " + url);
        try {
            //Note: already, this will (possibly, depending on errors) be overwritten later
            ServerBuilder serverBuilder = new ServerBuilder();
            serverBuilder.setDefaultComponentManagerUrn("urn:publicid:IDN+dummyforscan+authority+cm");
            serverBuilder.setName("scannedServer");
            if (trustedAlias != null) {
                LOG.debug("AM connection will trust SSL certificate alias " + trustedAlias);
                serverBuilder.setAllowedCertificateAlias(trustedAlias);
            } else {
                LOG.debug("AM connection will NOT trust any SSL certificate alias");
            }
            if (trustedAuthCerts != null && !trustedAuthCerts.isEmpty())
                serverBuilder.setCertificateChain(KeyUtil.x509certificateChainToPem(trustedAuthCerts));

            ServiceBuilder serviceBuilder = new ServiceBuilder();
            serviceBuilder.setApi(api.getId());
            serviceBuilder.setApiVersion(api.getVersion());
            serviceBuilder.setUrl(url.toExternalForm());
            serviceBuilder.setUrn("urn:publicid:IDN+dummyforscan+authority+cm");
            serverBuilder.addServiceBuilder(serviceBuilder);

            Server wizardCreatedServer = serverBuilder.create();

            //contact server using AM API(v2 but it doesn't really matter) and use GetVersion to get correct AM urls...
            AggregateManager2 am2 = new AggregateManager2(getLogger(), jFedPreferences);
            ConnectionBuilder connectionBuilder = connectionBuilderFactory.createConnectionBuilder();

            connectionBuilder.setDebugInfo(new JFedConnection.DebugInfo(wizardCreatedServer.getServices().get(0), null));
            connectionBuilder.setProxy(getProxyInfo(), false);
            connectionBuilder.setUrl(url);
            connectionBuilder.useHttps(new JFedTrustStore(wizardCreatedServer), null/*handleUntrustedCallback*/);
            connectionBuilder.useSslClientAuthentication(input.userCerts, input.userPrivateKey);

            AggregateManager2.AggregateManagerReply<AggregateManager2.VersionInfo> versionInfoReply =
                    am2.getVersion(connectionBuilder.buildSfaConnection());

            log("  Processing GetVersion result. (" + XmlRpcPrintUtil.xmlRpcObjectToString(versionInfoReply.getRawResult()).length() + " bytes in json)");//XmlRpcPrintUtil.xmlRpcObjectToString(versionInfoReply.getRawResult()));
            if (versionInfoReply.getRawResult() == null) {
                log("GetVersion result is null");
                return false;
            }
            if (versionInfoReply.getRawResult().get("value") == null) {
                log("GetVersion result has no value");
                return false;
            }
            if (!(versionInfoReply.getRawResult().get("value") instanceof Map)) {
                log("GetVersion result has non Map value: " + versionInfoReply.getRawResult().get("value").getClass().getName());
                return false;
            }
            Map valueHt = (Map) versionInfoReply.getRawResult().get("value");

            getVersionSuccess = true;

            ServerScanner.ScannedInformation<AuthorityType> scannedType =
                    AuthorityTypeFinder.findTypeFromGetVersionReply(versionInfoReply.getRawResult());
            if (scannedType != null) {
                types.addAll(scannedType);
                log("Type(s) derived from GetVersion: " + scannedType.getSet());
            } else
                log("No type derived from GetVersion reply");

            String newHrn = (String) valueHt.get("hrn");
            if (newHrn != null)
                serverBuilder.setName(newHrn);

            String newUrn = (String) valueHt.get("urn");
            if (newUrn != null) {
                serverBuilder.setDefaultComponentManagerUrn(newUrn);
                urns.add(ScanCertainty.RECV_FROM_SERVER, newUrn);
            }

            String amType = (String) valueHt.get("geni_am_type");
            if (amType != null) {
                for (AuthorityType t : AuthorityType.values()) {
                    if (t.name().equalsIgnoreCase(amType.trim())) {
                        serverBuilder.setServerType(amType);
                        types.add(ScanCertainty.RECV_FROM_SERVER, t);
                    }
                }
            }

            if (valueHt.get("orca_version") != null) {
                serverBuilder.setServerType(amType);
                types.add(ScanCertainty.RECV_FROM_SERVER, AuthorityType.ORCA);
            }

            ApiInfo.Api realServerType = null;
            if (valueHt.containsKey("geni_api")) {
                Object apiVersion = valueHt.get("geni_api");
                if (apiVersion instanceof Integer)
                    realServerType = new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, (Integer) apiVersion);
                if (apiVersion instanceof String)
                    realServerType =
                            new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, Integer.parseInt((String) apiVersion));
                if (realServerType != null)
                    addToUrlsByServerType(ScanCertainty.CONFIRMED, realServerType, url.toString());
            }

            Map geniApiVersions = (Map) valueHt.get("geni_api_versions");
            if (geniApiVersions.containsKey("3"))
                addToUrlsByServerType(ScanCertainty.RECV_FROM_SERVER, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3), fixUrl(url, (String) geniApiVersions.get("3")));
            if (geniApiVersions.containsKey("2"))
                addToUrlsByServerType(ScanCertainty.RECV_FROM_SERVER, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2), fixUrl(url, (String) geniApiVersions.get("2")));
            //ignore version 1, we don't understand it
            //if (geniApiVersions.containsKey("1"))
            //    addToUrlsByServerType(ScanCertainty.RECV_FROM_SERVER, new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 1), fixUrl(url, (String) geniApiVersions.get("1")));
            //TODO: check if received URL matches url tested.  (could be problem with ports or hostname)
//            if (confirmedUrlsByServerType.get(realServerType) == null || !confirmedUrlsByServerType.get(realServerType).contains(url.toString()))

            return true;
        } catch (Exception e) {
            log("exception calling GetVersion: " + e.getMessage(), e);
            return false;
        }
    }

    /**
     * returns true if call successful
     */
    public boolean callAMListResources() {
        log("calling AM ListResources");
        try {
            //first try to get credentials from user SA
            if (loggedInUser == null) {
                log("There is no logged in user: cannot call ListResources");
                return false;
            }
            UserAndSliceApiWrapper saWrapper =
                    new AutomaticUserAndSliceApiWrapper.BasicAutomaticUserAndSliceApiWrapperFactory(logger, geniUserProvider, connectionProvider, jFedPreferences).create();
            List<AnyCredential> credentialList = saWrapper.getUserCredentials(logger, loggedInUser.getUserUrn());
            if (credentialList.isEmpty()) {
                log("Failed to retrieve user credentials: cannot call ListResources");
                return false;
            }

            Map<ApiInfo.Api, URL> urlMap = new HashMap<>();
            for (Map.Entry<ApiInfo.Api, ScannedInformation<String>> e : urlsByApi.entrySet()) {
                ApiInfo.Api scannedInfoServerType = e.getKey();
                ScannedInformation<String> si = e.getValue();

                if (si != null && !si.isEmpty()) {
                    si.getFacts().stream()
                            .filter(fact -> fact.getCertainty().isMoreOrEqualCertainThan(ScanCertainty.RECV_FROM_SERVER))
                            .forEach(fact -> {
                                try {
                                    urlMap.put(scannedInfoServerType, new URL(fact.getFact()));
                                    log("callAMListResources is using URL " + fact.getFact() + " for " + scannedInfoServerType);
                                } catch (MalformedURLException e1) {
                                    log("callAMListResources is Ignoring malformed URL: " + fact.getFact());
                                }
                            });
                }
            }
            if (urlMap.isEmpty()) {
                log("callAMListResources has no URL to work on");
                return false;
            }

            ServerBuilder serverBuilder = new ServerBuilder();
            serverBuilder.setDefaultComponentManagerUrn("urn:publicid:IDN+dummyforscan+authority+cm");
            serverBuilder.setName("scannedServer");
            serverBuilder.setAllowedCertificateAlias(trustedAlias);
            if (trustedAuthCerts != null && !trustedAuthCerts.isEmpty())
                serverBuilder.setCertificateChain(KeyUtil.x509certificateChainToPem(trustedAuthCerts));

            for (Map.Entry<ApiInfo.Api, URL> e : urlMap.entrySet()) {
                ApiInfo.Api api = e.getKey();
                URL url = e.getValue();

                ServiceBuilder serviceBuilder = new ServiceBuilder();
                serviceBuilder.setApi(api.getId());
                serviceBuilder.setApiVersion(api.getVersion());
                serviceBuilder.setUrl(url.toExternalForm());
                serviceBuilder.setUrn("urn:publicid:IDN+dummyforscan+authority+cm");
                serverBuilder.addServiceBuilder(serviceBuilder);
            }

            Server wizardCreatedServer = serverBuilder.create();

            //contact server using AM API(v2 but it doesn't really matter) and use GetVersion to get correct AM urls...

            ConnectionBuilder connectionBuilder = connectionBuilderFactory.createConnectionBuilder();
            connectionBuilder.setDebugInfo(new JFedConnection.DebugInfo(wizardCreatedServer.getServices().get(0), null));
            connectionBuilder.setProxy(getProxyInfo(), false);
            connectionBuilder.useHttps(new JFedTrustStore(wizardCreatedServer), null/*handleUntrustedCallback*/);
            connectionBuilder.useSslClientAuthentication(input.userCerts, input.userPrivateKey);

            AbstractGeniAggregateManager.AggregateManagerReply<String> listResourcesReply = null;
            if (urlMap.containsKey(new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3))) {
                connectionBuilder.setUrl(urlMap.get(new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3)));

                AggregateManager3 am3 = new AggregateManager3(getLogger(), jFedPreferences);
                listResourcesReply =
                        am3.listResources(
                                connectionBuilder.buildSfaConnection(),
                                credentialList, "geni", "3",
                                false/*available*/, true/*compressed*/,
                                null/*extraOptions*/);
            }
            if (listResourcesReply == null && urlMap.containsKey(new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2))) {
                connectionBuilder.setUrl(urlMap.get(new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2)));

                AggregateManager2 am2 = new AggregateManager2(getLogger(), jFedPreferences);
                listResourcesReply =
                        am2.listResources(
                                connectionBuilder.buildSfaConnection(),
                                credentialList, "geni", "3",
                                false/*available*/, true/*compressed*/,
                                null/*sliceUrn*/,
                                null/*extraOptions*/);
            }
            if (listResourcesReply == null) {
                log("ListResources reply is null");
                return false;
            }
            log("  Processing ListResources result. (" + XmlRpcPrintUtil.xmlRpcObjectToString(listResourcesReply.getRawResult()).length() + " bytes in json)");//XmlRpcPrintUtil.xmlRpcObjectToString(versionInfoReply.getRawResult()));
            if (listResourcesReply.getRawResult() == null) {
                log("ListResources result is null");
                return false;
            }
            if (listResourcesReply.getRawResult().get("value") == null) {
                log("ListResources result has no value");
                return false;
            }
            if (!(listResourcesReply.getRawResult().get("value") instanceof String)) {
                log("ListResources result has non String value: " + listResourcesReply.getRawResult().get("value").getClass().getName());
                return false;
            }
            if (listResourcesReply.getValue() == null) {
                log("ListResources result has no processed value");
                return false;
            }
            String rspec = listResourcesReply.getValue();

            BasicStringRspec basicStringRspec = new BasicStringRspec(rspec);
            if (!basicStringRspec.isWellFormed()) {
                log("ListResources returned an RSpec that is not well formed");
                return false;
            }
            if (!basicStringRspec.isValidRspec()) {
                log("ListResources returned an RSpec that is not a valid RSpec");
                return false;
            }
            List<String> componentManagerUrns = basicStringRspec.getAllComponentManagerUrns();
            if (componentManagerUrns != null) {
                if (componentManagerUrns.size() == 1) {
                    String urn = componentManagerUrns.get(0);
//                    serverBuilder.setDefaultComponentManagerUrn(urn);
                    urns.add(ScanCertainty.RECV_FROM_SERVER, urn);
//                urns.add(ScanCertainty.INDIRECT_RECV_FROM_SERVER, urn);
                    log("ListResources found AM urn: " + urn);
                }
                if (componentManagerUrns.size() > 1) {
                    log("ListResources found multiple AM urn candidates: " + componentManagerUrns);
                    for (String urn : componentManagerUrns) {
//                        serverBuilder.setDefaultComponentManagerUrn(urn);
                        urns.add(ScanCertainty.GUESS, urn);
                    }
                }
                if (componentManagerUrns.isEmpty()) {
                    log("ListResources found not a single component_manager_urn");
                }
            } else {
                log("ListResources returned an RSpec that BasicStringRspec had trouble processing");
                return false;
            }

            return true;
        } catch (Exception e) {
            log("exception calling ListResources: " + e.getMessage(), e);
            return false;
        }
    }

    public void scan() {
        //no reason not to cache this for all sessions
//        failedServerPorts.clear();

        for (X509Certificate cert : input.trustedCerts)
            types.addAll(AuthorityTypeFinder.findTypeFromCert(cert));
        for (X509Certificate cert : input.userCerts)
            types.addAll(AuthorityTypeFinder.findTypeFromCert(cert));
        for (String urn : input.urns)
            types.addAll(AuthorityTypeFinder.findTypeFromUrn(urn));
        for (String url : input.urls)
            types.addAll(AuthorityTypeFinder.findTypeFromUrl(url));
        if (input.type != null)
            types.add(ScanCertainty.USER_INPUT, input.type);
        if (input.urns.size() == 1)
            urns.add(ScanCertainty.USER_INPUT, input.urns.get(0));

        log("Derived Authority type from initial input: " + types);

        if (mode == Mode.EXPERIMENTAL) {
            findServerBases();
            log("serverBases: " + serverBases);
        }

        input.urls.forEach(this::processInputUrl);
        if (mode == Mode.EXPERIMENTAL)
            types.getSet().forEach(this::addUrlsForType);

        //make copy of URLs
        Map<ApiInfo.Api, List<String>> origUrlsByServerType = new HashMap<>();
        for (Map.Entry<ApiInfo.Api, ScannedInformation<String>> e : urlsByApi.entrySet()) {
            ApiInfo.Api api = e.getKey();
            ScannedInformation<String> urls = e.getValue();
            origUrlsByServerType.put(api, new ArrayList<>(urls.getSet()));
        }

        for (Map.Entry<ApiInfo.Api, List<String>> e : origUrlsByServerType.entrySet()) {
            ApiInfo.Api serverType = e.getKey();
            List<String> urls = e.getValue();
            testUrls(serverType, urls);
        }

        if (mode == Mode.EXPERIMENTAL)
            if (types.isEmpty()) {
                log("No types are known, so falling back to trying all types. This could take a while.");
                for (AuthorityType authorityType : AuthorityType.values())
                    addUrlsForType(authorityType, ScanCertainty.FALLBACK);
            }

        Map<ApiInfo.Api, List<String>> newUrlsByServerType = new HashMap<>();
        for (Map.Entry<ApiInfo.Api, ScannedInformation<String>> e : urlsByApi.entrySet()) {
            ApiInfo.Api api = e.getKey();
            ScannedInformation<String> urls = e.getValue();
            List<String> orig_urls = origUrlsByServerType.get(api);
            List<String> newUrlList = new ArrayList<>(urls.getSet());
            if (orig_urls != null)
                newUrlList.removeAll(orig_urls);
            if (!newUrlList.isEmpty())
                newUrlsByServerType.put(api, newUrlList);
        }

        for (Map.Entry<ApiInfo.Api, List<String>> e : newUrlsByServerType.entrySet()) {
            ApiInfo.Api api = e.getKey();
            List<String> urls = e.getValue();
            testUrls(api, urls);
        }
    }

    private void testUrls(ApiInfo.Api api, List<String> urls) {
        log("URLs to test for " + api + ": " + urls);

        assert mode != null;
        assert urls != null;

//        assert mode == Mode.EXPERIMENTAL || urls.size() <= 1 : "mode="+mode+" urls.size()="+urls.size()+" urls="+urls;

        for (String urlStr : urls) {
            URL url;
            try {
                url = new URL(urlStr);
            } catch (MalformedURLException e1) {
                log("Invalid URL: " + urlStr);
                continue;
            }

            ServerPort serverPort =
                    new ServerPort(url.getHost(), url.getPort() <= 0 ? url.getDefaultPort() : url.getPort());
            boolean ok;
            if (failedServerPorts.contains(serverPort)) {
                log("Server port is already known not to work, will not try again: " + urlStr);
                ok = false;
            } else
                ok = checkUrl(urlStr);
            if (!ok) {
                log("Could not setup secure SSL connection to: " + urlStr);
                failedServerPorts.add(serverPort);
                continue;
            }

            switch (api.getName()) {
//                    case PlanetLabSliceRegistry: { triedGetVersion = true; successGetVersion = callPlanetLabSliceRegistryGetVersion(api, url); break; }
//                    case PROTOGENI_SA: { triedGetVersion = true; successGetVersion = callProtoGeniSAGetVersion(api, url); break; }
                case GENI_AM: {
                    if (callAMGetVersion(api, url)) {
                        boolean goodUrnKnown = (!urns.isEmpty()) &&
                                urns.getMostTrustedScannedSingleFact().getCertainty().isMoreOrEqualCertainThan(ScanCertainty.RECV_FROM_SERVER);

                        if (goodUrnKnown) {
                            log("AM URN known already, no need for ListResources.");
                        } else {
                            log("No AM URN known, will try to get it using ListResources.");
                            final boolean listResourcesResult = callAMListResources();
                            if (!listResourcesResult)
                                log("ListResources was unsuccesful");
                        }
                    }
                    break;
                }
//                    case GENI_CH_SA: { triedGetVersion = true; successGetVersion = callFederationSAOrMAGetVersion(api, url); break; }
//                    case GENI_CH_MA: { triedGetVersion = true; successGetVersion = callFederationSAOrMAGetVersion(api, url); break; }
                case PROTOGENI_CH:
                case GENI_CH:
                case GENI_SCS:
                case GTS_API: {
                    break;
                }
                default: {
                    log("unsupported servertype: " + api);
                    break;
                }
            }
        }

    }


    public AuthorityType getType() {
        return types.getMostTrusted();
    }

    //    private Testbed autoResult = null;
    public Testbed getAutoResult() {
//        if (autoResult == null) {
//            try {
//                autoResult = createResult(types, urlsByApi, hrns, urns, trustedAuthCerts, trustedAlias);
//            } catch (JFedException e) {
//                log("Error creating result: " + e.getMessage(), e);
//            }
//        }
//        return autoResult;
        return createResult(types, urlsByApi, hrns, urns, trustedAuthCerts, trustedAlias);
    }

    public synchronized void fireLogLine(String logLine) {
        for (AuthorityScannerLogListener l : resultListeners)
            l.onLogLine(logLine);
    }

    public synchronized void addResultListener(AuthorityScannerLogListener l) {
        resultListeners.add(l);
    }

    public synchronized void removeResultListener(AuthorityScannerLogListener l) {
        resultListeners.remove(l);
    }

    public String getLogText() {
        String res = "";
        for (String logline : logs)
            res += logline + "\n";
        return res;
    }

    public List<String> getLogs() {
        return logs;
    }

    public ScannedInformation<String> getHrns() {
        return hrns;
    }

    public ScannedInformation<String> getUrns() {
        return urns;
    }

    public Map<ApiInfo.Api, ScannedInformation<String>> getUrlsByApi() {
        return urlsByApi;
    }

    public String getTrustedAlias() {
        return trustedAlias;
    }

    public Set<X509Certificate> getTrustedAuthCerts() {
        return trustedAuthCerts;
    }

    public ScannedInformation<AuthorityType> getTypes() {
        return types;
    }

    public boolean isConnectionSuccess() {
        return connectionSuccess;
    }

    public boolean isGetVersionSuccess() {
        return getVersionSuccess;
    }

    public enum ScanCertainty {
        WRONG(-1),                       //Known to be false, but might be close to truth
        FALLBACK(0),                     //wild guess, derived from other data, licely not correct. But possibly better than nothing.
        GUESS(1),                        //derived from other data (may be incorrect)
        GOOD_GUESS(2),                   //derived from other data, likely correct (may still be incorrect)
        USER_INPUT(3),                   //originally inputted by user
        INDIRECT_RECV_FROM_SERVER(4),    //derived from server date, such as certificate (but server could be wrong)
        RECV_FROM_SERVER(5),             //received from server, (but server could be wrong)
        CONFIRMED(10);                   //confirmed to be correct

        public final int value;

        ScanCertainty(int value) {
            this.value = value;
        }

        public boolean isMoreCertainThan(ScanCertainty o) {
            return value > o.value;
        }

        public boolean isMoreOrEqualCertainThan(ScanCertainty o) {
            return value >= o.value;
        }
    }

    public static class ScannedSingleFact<T> {
        private final T fact;
        private final ScanCertainty certainty;

        public ScannedSingleFact(ScanCertainty certainty, T fact) {
            this.fact = fact;
            this.certainty = certainty;
        }

        public T getFact() {
            return fact;
        }

        public ScanCertainty getCertainty() {
            return certainty;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            ScannedSingleFact that = (ScannedSingleFact) o;

            if (certainty != that.certainty) return false;
            if (fact != null ? !Objects.equals(fact, that.fact) : that.fact != null) return false;
            return true;
        }

        @Override
        public int hashCode() {
            int result = fact != null ? fact.hashCode() : 0;
            result = 31 * result + (certainty != null ? certainty.hashCode() : 0);
            return result;
        }
    }

    public static class ScannedInformation<T> {
        final Set<ScannedSingleFact<T>> infoSet = new HashSet<>();

        public void add(ScanCertainty certainty, T fact) {
            infoSet.add(new ScannedSingleFact<>(certainty, fact));
        }

        public void add(ScannedSingleFact<T> fact) {
            infoSet.add(fact);
        }

        /**
         * return null if nothing known
         */
        public T getMostTrusted() {
            ScannedSingleFact<T> res = getMostTrustedScannedSingleFact();
            if (res == null) return null;
            return res.getFact();
        }

        /**
         * return null if nothing known
         */
        public ScannedSingleFact<T> getMostTrustedScannedSingleFact() {
            ScannedSingleFact<T> res = null;
            for (ScannedSingleFact<T> fact : infoSet) {
                if (res == null || fact.certainty.isMoreCertainThan(res.certainty))
                    res = fact;
            }
            if (res == null)
                return null;
            return res;
        }

        public Set<T> getSet() {
            return infoSet.stream().map(ScannedSingleFact::getFact).collect(Collectors.toSet());
        }

        public Collection<ScannedSingleFact<T>> getFacts() {
            return infoSet;
        }

        public boolean isEmpty() {
            return infoSet.isEmpty();
        }

        public void addAll(ScannedInformation<T> infos) {
            if (infos == null) return;
            infoSet.addAll(infos.infoSet);
        }

        @Override
        public String toString() {
            return infoSet.toString();
        }
    }

    //serverBase: hostname:port  or hostname:port/bla/bla/       port == 0 means unknown port   (known default ports (80,443) should be added)
    private class ServerBase {
        final String hostname;
        final int port;
        final String path;

        ServerBase(String hostname, int port, String path) {
            assert hostname != null;
            this.hostname = hostname;
            this.port = port;
            this.path = path;
            assert !hostname.contains(":") : "ServerBase hostname contains port: \"" + hostname + "\"";
        }

        @Override
        public String toString() {
            if (path == null) {
                if (port <= 0)
                    return hostname + "/";
                else
                    return hostname + ':' + port + "/";
            } else {
                if (port <= 0)
                    return hostname + "/" + path + '/';
                else
                    return hostname + ':' + port + "/" + path + '/';
            }
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            ServerBase that = (ServerBase) o;

            if (port != that.port) return false;
            if (!Objects.equals(hostname, that.hostname)) return false;
            if (path != null ? !Objects.equals(path, that.path) : that.path != null) return false;

            return true;
        }

        @Override
        public int hashCode() {
            int result = hostname.hashCode();
            result = 31 * result + port;
            result = 31 * result + (path != null ? path.hashCode() : 0);
            return result;
        }
    }

    private class ServerPort {
        public final String hostname;
        public final int port;

        private ServerPort(String hostname, int port) {
            this.hostname = hostname;
            this.port = port;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            ServerPort that = (ServerPort) o;

            if (port != that.port) return false;
            if (!Objects.equals(hostname, that.hostname)) return false;

            return true;
        }

        @Override
        public int hashCode() {
            int result = hostname.hashCode();
            result = 31 * result + port;
            return result;
        }
    }


    public enum Mode {BASIC, EXPERIMENTAL}

    public void setMode(Mode mode) {
        this.mode = mode;
    }
}
