package be.iminds.ilabt.jfed.lowlevel.api.test;

import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Server;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnection;
import be.iminds.ilabt.jfed.lowlevel.ssh_key_info.SshFilesKeyInfo;
import be.iminds.ilabt.jfed.rspec.model.BasicStringRspec;
import be.iminds.ilabt.jfed.rspec.model.ModelRspec;
import be.iminds.ilabt.jfed.rspec.model.ModelRspecType;
import be.iminds.ilabt.jfed.rspec.model.RspecNode;
import be.iminds.ilabt.jfed.rspec.rspec_source.ManifestRspecSource;
import be.iminds.ilabt.jfed.ssh_terminal_tool.unix.OpenSshKnownHostsFile;
import be.iminds.ilabt.jfed.testing.base.ApiTest;
import be.iminds.ilabt.jfed.util.*;
import be.iminds.ilabt.jfed.util.tmp_file_helpers.TmpContentFile;
import be.iminds.ilabt.jfed.util.tmp_file_helpers.TmpFile;
import net.schmizz.sshj.SSHClient;
import net.schmizz.sshj.connection.channel.direct.Session;
import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
import net.schmizz.sshj.userauth.keyprovider.OpenSSHKeyFile;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.interfaces.RSAPrivateKey;
import java.util.*;

/**
 * NodeLoginTester: helper class that specifically tests node login.
 */
public class NodeLoginTester {
    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(NodeLoginTester.class);

    private ApiTest test;

    private SSHKeyHelper sshKeyHelper;
    private String sshUsername;
    private String sshHostname;
    private int sshPort;
    private JFedConnection.SshProxyInfo manifestSshProxy;
    private JFedConnection.SshProxyInfo sshProxy;

    private boolean parsed = false;

    private long nodeLoginDeadlineMs = 20 * 60 * 1000L;
    private String preferredUser = null;
    private Integer nodeLoginForceIpVersion = null;

    public NodeLoginTester(ApiTest test, String fixedSshPublicKeyFile, String fixedSshPrivateKeyFile, String pass) throws NoSuchAlgorithmException {
        this.test = test;

        if ((fixedSshPublicKeyFile == null) != (fixedSshPrivateKeyFile == null))
            throw new RuntimeException("fixedSshPublicKeyFile and fixedSshPrivateKeyFile must either both be null, or both differ from null");

        if (fixedSshPrivateKeyFile != null) {
            assert fixedSshPublicKeyFile != null;
            try {
                String privatePem = IOUtils.fileToString(fixedSshPrivateKeyFile);
                String publicFileContent = IOUtils.fileToString(fixedSshPublicKeyFile);

                boolean isCertificate = KeyUtil.hasX509Certificate(publicFileContent);
                boolean isOpenSsl = KeyUtil.isOpenSshRsaKey(publicFileContent);

                PublicKey publicKey = null;
                if (isOpenSsl)
                    publicKey = KeyUtil.openSshAuthorizedKeysFormatRsaPublicKey(publicFileContent);
                if (isCertificate)
                    publicKey = KeyUtil.pemToX509Certificate(publicFileContent).getPublicKey();
                assert publicKey != null;

                RSAPrivateKey privateKey =
                        KeyUtil.pemToRsaPrivateKey(privatePem, pass == null ? null : pass.toCharArray());
                this.sshKeyHelper = new SSHKeyHelper(publicKey, privateKey);
            } catch (Exception e) {
                LOG.error("Exception while reading key: " + e.getMessage(), e);
                throw new RuntimeException("Exception while reading key: " + e.getMessage(), e);
            }
        } else
            this.sshKeyHelper = new SSHKeyHelper(); //random
    }

    public NodeLoginTester(ApiTest test, SSHKeyHelper existingSSHKeyHelper) throws NoSuchAlgorithmException {
        assert test != null;
        assert existingSSHKeyHelper != null;

        this.test = test;
        this.sshKeyHelper = existingSSHKeyHelper;
    }


    private NodeLoginTester(NodeLoginTester o) {
        this.test = o.test;
        this.sshKeyHelper = o.sshKeyHelper;
        this.sshUsername = o.sshUsername;
        this.sshHostname = o.sshHostname;
        this.sshPort = o.sshPort;
        this.sshProxy = o.sshProxy;
        this.manifestSshProxy = o.manifestSshProxy;
        this.parsed = o.parsed;
        this.preferredUser = o.preferredUser;
        this.nodeLoginDeadlineMs = o.nodeLoginDeadlineMs;
    }

    public NodeLoginTester copy() {
        return new NodeLoginTester(this);
    }

    public boolean hasParsed() {
        return parsed;
    }

    public void setSshProxy(JFedConnection.SshProxyInfo sshProxy) {
        this.sshProxy = sshProxy;
    }

    public void setNodeLoginDeadlineMs(long nodeLoginDeadlineMs) {
        this.nodeLoginDeadlineMs = nodeLoginDeadlineMs;
    }

    public void setNodeLoginPreferredUser(String nodeLoginPreferredUser) {
        this.preferredUser = nodeLoginPreferredUser;
    }

    /**
     * parse the login info for the first node found on the specified authority
     *
     * @param rspec    the rspec with login info
     * @param server   the authority for which to find node login info. if null, a node from any authority is ok.
     * @param clientId the clientId of the target node. May be null, in that case any node will be used.
     * @returns true if successfully found node login info, false otherwise.
     */
    public boolean parseSshInfoFromGeni3ManifestRspec(String rspec, Server server, String clientId) {
        if (rspec == null) return false;
        test.assertNotNull(rspec, "Rspec is null");

        //quick HACK to allow the planetlab Europe rspec manifest
        // FEDIBBTDEV-113, bvermeul, copied this nice stuff from suite/rspec-edit/src/main/java/be/iminds/ilabt/jfed/ui/rspeceditor/model/Rspec.java
        if (rspec.contains("<RSpec")) {
            test.note("We received an SFA RSpec instead of a GENI RSpec v3, but we'll try to work around.");
            //LOG.debug("HACK: rewriting received planetlab manifest XML to make it parsable xml");
            rspec =
                    rspec.replace("<RSpec type=\"SFA\"", "<rspec xmlns=\"http://www.geni.net/resources/rspec/3\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" type=\"manifest\"");
            rspec =
                    rspec.replace("<RSpec ", "<rspec xmlns=\"http://www.geni.net/resources/rspec/3\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" type=\"manifest\"");
            rspec = rspec.replace("</RSpec>", "</rspec>");
            rspec = rspec.replace("<network name=\"ple\">", "");
            rspec = rspec.replace("</network>", "");
            //LOG.debug("HACK: rewriten xml: "+rspec);
        }

        BasicStringRspec basicStringRspec = new BasicStringRspec(rspec);

        List<BasicStringRspec.BasicNodeInfo> basicNodeInfos = basicStringRspec.getBasicNodeInfo();
        parsed = true;
        List<BasicStringRspec.BasicNodeInfo> basicUsableNodeInfos = new ArrayList<>();
        for (BasicStringRspec.BasicNodeInfo basicNodeInfo : basicNodeInfos) {
            if (server != null && server.getDefaultComponentManagerUrn() != null &&
                    (basicNodeInfo.getComponentManagerId() == null || !Objects.equals(basicNodeInfo.getComponentManagerId(), server.getDefaultComponentManagerUrn())))
                continue;

            if (clientId != null && basicNodeInfo.getClientId() != null && !Objects.equals(basicNodeInfo.getClientId(), clientId))
                continue;

            //note that normally, client_id would be required. But we PLE doesn't provide it (which is a bug), so we fall back to component_id there.
            if (basicNodeInfo.getClientId() != null ||
                    basicNodeInfo.getComponentId() != null)
                basicUsableNodeInfos.add(basicNodeInfo);
        }

        int nodeCount = basicUsableNodeInfos.size();

        if (nodeCount > 0) {
            BestNodeLoginFinder.Feedback feedback = new BestNodeLoginFinder.Feedback() {
                @Override
                public void info(String s) {
                    test.note(s);
                }

                @Override
                public void error(String s) {
                    test.errorNonFatal(s);
                }

                @Override
                public void debug(String s) {
                    LOG.debug(s);
                }
            };
            BestNodeLoginFinder finder = new BestNodeLoginFinder(
                    basicStringRspec,
                    preferredUser,
                    test.getUser(),
                    feedback);
            //use first node if multiple available
            BasicStringRspec.LoginService best = finder.findBestLogin(
                    basicUsableNodeInfos.get(0).getUniqueId());

            if (best != null) {
                sshUsername = best.getUsername();
                sshHostname = best.getHostname();
                sshPort = best.getPort();
                manifestSshProxy = best.getSshProxy();
                return true;
            } else
                return false;
        } else {
            test.note("Found no nodes (with a client_id) in manifest RSpec" + (server == null ? ""
                    : " (for nodes of auth " + server.getDefaultComponentManagerUrn() + ")"));
            return false;
        }

//        //parse the RSpec XML (this manual parsing doesn't look at which node, so it allows errors with the manifest RSpec that cause problems in real scenarios)
//        parsed = true;
//        int nodeCount = 0;
//        int loginCount = 0;
//        try {
//            Class docClass = be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.RSpecContents.class;
//            String packageName = docClass.getPackage().getName();
//            JAXBContext jc = JAXBContext.newInstance(packageName);
//            Unmarshaller u = jc.createUnmarshaller();
//            JAXBElement<RSpecContents> doc = (JAXBElement<be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.RSpecContents>) u.unmarshal( new StringReader(rspec));
//            be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.RSpecContents c = doc.getValue();
//
//            be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.RspecTypeContents typ = c.getType();
//
//            for (Object o : c.getAnyOrNodeOrLink()) {
//                if (o instanceof JAXBElement) {
//                    JAXBElement el = (JAXBElement) o;
//                    if (el.getValue() instanceof be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.NodeContents) {
//                        be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.NodeContents node = (be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.NodeContents) el.getValue();
//
//                        String nodeName = node.getClientId();
//                        nodeCount++;
//
//                        for (Object nodeElO : node.getAnyOrRelationOrLocation()) {
//                            if (nodeElO instanceof JAXBElement) {
//                                JAXBElement nodeEl = (JAXBElement) nodeElO;
//                                if (nodeEl.getValue() instanceof be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.ServiceContents) {
//                                    be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.ServiceContents serviceC = (be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.ServiceContents) nodeEl.getValue();
//                                    for (Object serviceElO : serviceC.getAnyOrLoginOrInstall()) {
//                                        if (serviceElO instanceof JAXBElement) {
//                                            JAXBElement serviceEl = (JAXBElement) serviceElO;
//                                            if (serviceEl.getValue() instanceof be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.LoginServiceContents) {
//                                                be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.LoginServiceContents loginSC = (be.iminds.ilabt.jfed.rspec.manifest.geni_rspec_3.LoginServiceContents) serviceEl.getValue();
//
//                                                String auth = loginSC.getAuthentication();
//
//                                                String aHostname = loginSC.getHostname();
//                                                int aPort = 0;
//                                                if (loginSC.getPort() != null)
//                                                    aPort = Integer.parseInt(loginSC.getPort());
//                                                String aUsername = loginSC.getUsername();
//
//                                                if (auth.equals("ssh-keys")) {
//                                                    test.assertEquals("ssh-keys", auth, "service login authentication must be ssh-keys for node " + nodeName);
//                                                    test.assertNotNull(aHostname, "no hostname in service login for node " + nodeName);
//                                                    test.assertNotNull(aUsername, "no username in service login for node " + nodeName);
//                                                    //TODO add support for missing username?
//
//                                                    hostname = aHostname;
//                                                    port = aPort;
//                                                    username = aUsername;
//
//                                                    loginCount++;
//                                                } else {
//                                                    test.note("Unsupported auth in manifest RSpec service login: " + auth);
//                                                }
//                                            }
//                                        }
//                                    }
//                                }
//                            }
//                        }
//                    }
//                }
//            }
//        } catch (JAXBException e) {
//            throw new RuntimeException("Exception parsing manifest RSpec Xml: "+e.getMessage(), e);
//        }
//
//        if (loginCount > 0)
//        {
//            test.assertTrue(loginCount > 0, "no service login found (" + loginCount + " service logins for " + nodeCount + " nodes)");
//            test.assertNotNull(hostname);
//            test.assertNotNull(username);
//
//            if (loginCount > 1)
//                test.note("Found multiple node logins in manifest rspec (" + loginCount + " service logins for " + nodeCount + " nodes), using last one.");
//
//            sshUsername = username;
//            sshHostname= hostname;
//            sshPort = port;
//            return true;
//        } else {
//            test.note("Found no node service login in manifest RSpec.");
//            return false;
//        }
    }

//    public void setSshInfo(String sshUsername, String sshHostname, int sshPort, sshProxy) {
//        this.sshUsername = sshUsername;
//        this.sshHostname = sshHostname;
//        this.sshPort = sshPort;
//    }


    public void testNodeLogin(boolean warnIfNotYetReady) throws IOException {
        testNodeLogin(warnIfNotYetReady, true);
    }
    public void testNodeLogin(boolean warnIfNotYetReady, boolean internalSsh) throws IOException {
        assert parsed : "No RSpec was parsed for node login info";
        if (!parsed)
            test.skip("No manifest RSpec parsed yet, so SSH login cannot be tested.");

        if (sshHostname == null)
            test.skip("No node / service / login in manifest RSpec, so SSH login cannot be tested.");

        if (sshProxy != null) {
            test.note("Using proxy private key matching public key: " +
                    KeyUtil.publicKeyToOpenSshAuthorizedKeysFormat(
                            sshProxy.getSshKeyInfo().getPublicKey()
                    )
            );
        }

        if (nodeLoginForceIpVersion != null) {
            boolean skip = false;
            if (nodeLoginForceIpVersion != 4 && nodeLoginForceIpVersion != 6) {
                test.warn("nodelogin_force_ip_version property is set to invalid int: " + nodeLoginForceIpVersion + ". Supported: 4 or 6");
                skip = true;
            }


            //only simple checks for IPv4 and Ipv6:
            //  Needs to capture al valid Ipv4 and Ipv6, but may have many false positive for this purpose here.
            //  TODO: could also use org.xbill.DNS.Address.getByAddress or isDottedQuad
            if (sshHostname.matches("[0-9.]+")) {
                if (nodeLoginForceIpVersion == 4)
                    test.note("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", and SSH hostname is already IPv4.");
                else
                    test.warn("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", but SSH hostname is IPv4: Will use IPv4 anyway!");
                skip = true;
            }
            if (sshHostname.matches("[0-9A-fa-f:]+")) {
                if (nodeLoginForceIpVersion == 6)
                    test.note("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", and SSH hostname is already IPv6.");
                else
                    test.warn("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", but SSH hostname is IPv6: Will use IPv6 anyway!");
                skip = true;
            }

            if (!skip) {
                boolean foundIp = false;
                String origSshHostname = sshHostname;
                InetAddress[] addresses = org.xbill.DNS.Address.getAllByName(sshHostname);
                for (InetAddress address : addresses) {
                    if (nodeLoginForceIpVersion == 4 && org.xbill.DNS.Address.familyOf(address) == org.xbill.DNS.Address.IPv4) {
                        sshHostname = address.getHostAddress();
                        foundIp = true;
                        break;
                    }
                    if (nodeLoginForceIpVersion == 6 && org.xbill.DNS.Address.familyOf(address) == org.xbill.DNS.Address.IPv6) {
                        sshHostname = address.getHostAddress();
                        foundIp = true;
                        break;
                    }
                }
                if (!foundIp) {
                    test.warn("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", " +
                            "but did not find an IPv" + nodeLoginForceIpVersion + " address for the SSH hostname (\"" + sshHostname + "\")." +
                            " All found addresses: " + Arrays.asList(addresses));
                } else {
                    test.note("nodelogin_force_ip_version property is set to IPv" + nodeLoginForceIpVersion + ", " +
                            "and found an IPv" + nodeLoginForceIpVersion + " address (" + sshHostname + ") for the SSH hostname (\"" + origSshHostname + "\")");
                }
            }
        }

        if (internalSsh)
            testNodeLoginInternalSsh(warnIfNotYetReady);
        else
            testNodeLoginExternalSsh(warnIfNotYetReady);
    }

    private List<String> replace(List<String> orig, String match, String replacement) {
        List<String> res = new ArrayList<>();
        for (String o : orig) {
            res.add(o.replace(match, replacement));
        }
        return res;
    }

    private void testNodeLoginExternalSsh(boolean warnIfNotYetReady) throws IOException {
        // bvermeul, wait max 20 minutes till ssh login works (as PLE says that sliver is ready while it is not)
        // FEDIBBTDEV-113
        long now = System.currentTimeMillis();
        long start = now;
        long deadline = now + nodeLoginDeadlineMs;


        String remoteCommand = "ls /";

        //for SSH proxy to work, we need to store the private key unencrypted.
        final List<TmpFile> tmpFiles = new ArrayList<>();
        List sshCommandList = new ArrayList<>(Arrays.asList("ssh", "-i", "$i", "$u@$h",  "-oPort=$p",
                "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no"));

        if (sshProxy != null) {
            assert sshProxy instanceof JFedConnection.SshProxyInfo;

            JFedConnection.SshProxyInfo sshProxyInfo = sshProxy;

            //store proxyOpenSshKey
            if (sshProxyInfo.getHostKey() != null) {
                //TODO: not ideal, as this should lock to prevent simultanious access to the file.
                //not an issue though, as it will normally never write anyway...
                OpenSshKnownHostsFile openSshKnownHostsFile = new OpenSshKnownHostsFile();
                openSshKnownHostsFile.addKnownHost(sshProxyInfo.getHostname(), sshProxyInfo.getHostKey());
            }

            sshCommandList.addAll(Collections.singletonList("-oProxyCommand=\"ssh -i '$I' -oPort=$P $U@$H -W %h:%p\""));

            sshCommandList = replace(sshCommandList, "$P", sshProxyInfo.getPort() + "");
            sshCommandList = replace(sshCommandList, "$H", sshProxyInfo.getHostname());
            sshCommandList = replace(sshCommandList, "$U", sshProxyInfo.getUsername());

            if (sshProxyInfo.getSshKeyInfo() != null) {
                assert sshProxyInfo.getSshKeyInfo() instanceof SshFilesKeyInfo :
                        "sshProxyInfo.getSshKeyInfo() -> "+sshProxyInfo.getSshKeyInfo().getClass().getName();
                SshFilesKeyInfo sshFilesKeyInfo = (SshFilesKeyInfo)sshProxyInfo.getSshKeyInfo();

                TmpFile tmpFile = sshFilesKeyInfo.getUnencryptedPrivateKeyFile();
                tmpFiles.add(tmpFile);
                //this assumes: unencrypted key file
                sshCommandList = replace(sshCommandList, "$I", tmpFile.getFilename());
            }
        }

        TmpFile tmpPrivSshKeyFile = new TmpContentFile("testprivkey", "pem",
                new String(KeyUtil.privateKeyToAnyPem(sshKeyHelper.getSshPrivateKey())),
                true /*assureUserOnlyPermissions*/
                );
        tmpFiles.add(tmpPrivSshKeyFile);

        sshCommandList = replace(sshCommandList, "$p", sshPort + "");
        sshCommandList = replace(sshCommandList, "$h", sshHostname);
        sshCommandList = replace(sshCommandList, "$u", sshUsername);
        //this assumes: unencrypted key file, and only 1 key file
        sshCommandList = replace(sshCommandList, "$i", tmpPrivSshKeyFile.getFilename());

        sshCommandList.add(remoteCommand);

        for (TmpFile tmpFile : tmpFiles) {
            tmpFile.store();
        }


        test.note("Using SSH commandList: "+sshCommandList);

        while (now <= deadline) {
            //Test node login using SSH private key
            test.note("Trying to connect to " + sshHostname + ":" + sshPort + " as user \"" + sshUsername + "\"");

            try {
                ProcessBuilder pb = new ProcessBuilder(sshCommandList);
                pb.redirectErrorStream(true);

                Process p = pb.start();
                //                    Process p = Runtime.getRuntime().exec(finalCommand);
                BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));

                String line;
                String allLines = "";
                line = input.readLine();
                LOG.debug("Read line from SSH command: \""+line+"\"");
                while (line != null) {
                    allLines += line + "\n";
                    line = input.readLine();
                    LOG.debug("Read line from SSH command: \""+line+"\"");
                }
                //note: input might close before terminal stops.
                input.close();
                try {
                    p.waitFor();
                } catch (InterruptedException e) {
                    LOG.warn("ExternalTerminal Thread: processOutputMonitor thread interrupted while waiting for terminal to stop", e);
                }
                int exitVal = p.exitValue();
                LOG.info("SSH command exited with exit value (must be 0, certainly not 255): " +exitVal);
                test.note("SSH command exited with exit value (must be 0, certainly not 255): " +exitVal);

                //exit value: exit value of command, or 255 on error
                //ls exit value: 0 on success

                test.note("SSH call result: \n" + allLines, true/*formatAsCode*/);

                boolean testSuccess = exitVal == 0 && allLines.length() > 5;

                if (testSuccess) {
                    test.assertTrue(p.exitValue() != 255, "I executed \"" + remoteCommand + "\" on the remote host, and expected a non 255 exit value. Instead I got " + p.exitValue() + "");
                    test.assertTrue(p.exitValue() == 0, "I executed \"" + remoteCommand + "\" on the remote host, and expected a 0 exit value. Instead I got " + p.exitValue()+ "");
                    test.assertTrue(allLines.length() > 5, "I executed \"" + remoteCommand + "\" on the remote host, and expected some reply. Instead I got: \"" + allLines.trim() + "\".");

                    for (TmpFile tmpFile : tmpFiles) {
                        tmpFile.delete();
                    }

                    return;
                } else {
                    test.note("Login attempt failed");
                }
            } catch (Exception e) {
                LOG.error("Unhandled Exception in ExternalTerminal Thread.", e);
            } catch (AssertionError e) {
                LOG.error("Unhandled AssertionError in ExternalTerminal Thread.", e);
            }


            test.note("Timing info: failed login attempt took " + (System.currentTimeMillis() - now) + " ms");

            if (warnIfNotYetReady)
                test.warn("SSH connection not successful (should be available from the moment that status is 'ready').");
            else
                test.note("SSH connection not successful (should be available from the moment that status is 'ready').");
            now = System.currentTimeMillis();
            if (now <= deadline)
                try {
                    long sleepTimeMs = 30000L;
                    if (deadline - now < sleepTimeMs) {
                        sleepTimeMs = deadline - now;

                        //give it enough room to be able to restart one more time
                        if (sleepTimeMs > 1000L)
                            sleepTimeMs -= 1000L;
                    }

                    test.note("Trying again in " + (sleepTimeMs / 1000) + " seconds...");

                    Thread.sleep(sleepTimeMs);
                } catch (InterruptedException e) { /* Ignore*/ }
            now = System.currentTimeMillis();
        }

        for (TmpFile tmpFile : tmpFiles) {
            tmpFile.delete();
        }

        test.fatalError("Could not login to host: connection setup, authentication with private key or remote command failed.");
    }

    private void testNodeLoginInternalSsh(boolean warnIfNotYetReady) throws IOException {
        // bvermeul, wait max 20 minutes till ssh login works (as PLE says that sliver is ready while it is not)
        // FEDIBBTDEV-113
        long now = System.currentTimeMillis();
        long start = now;
        long deadline = now + nodeLoginDeadlineMs;
        boolean isAuthenticated = false;
        final SSHClient ssh = new SSHClient();
        ssh.addHostKeyVerifier(new PromiscuousVerifier()); //don't verify host keys

        while (now <= deadline) {
            //Test node login using SSH private key

            test.note("Trying to connect to " + sshHostname + ":" + sshPort + "   (will later try to authenticate as user \"" + sshUsername + "\")");

            boolean forceSshProxyDns = false;
            if (sshProxy != null) {
                //force target address, so that we don't have to rely on local resolution
                ssh.setSocketFactory(SshProxySocketFactory.createWithForcedTarget(sshProxy, sshHostname, sshPort));
                forceSshProxyDns = true;
                test.note("Will Use SSH proxy for all SSH node login attempts that follow. info: " + sshProxy);
            }

            boolean isConnected = false;
            try {
                //TODO: add fix to sshj so this is cleaner
                if (forceSshProxyDns)
                    ssh.connect("localhost", sshPort); //prevent DNS lookup failures (will still connect to correct node thanks to SSH proxy)
                else
                    ssh.connect(sshHostname, sshPort);
                isConnected = true;
            } catch (IOException e) {
                if (warnIfNotYetReady)
                    test.warn("Exception while setting up SSH connection: " + e.getMessage(), e);
                else
                    test.note("Exception while setting up SSH connection: " + e.getMessage(), e);
                isAuthenticated = false;
            }

            if (isConnected)
                try {
                    String privateKeyToPrint = new String(sshKeyHelper.getPEMAnyPrivateKey());
                    if (privateKeyToPrint.length() > 75)
                        privateKeyToPrint = privateKeyToPrint.substring(0, 75) + " ...";
                    test.note("Connected to target node. Authenticating as " + sshUsername + " with PEM private key matching public key: " + sshKeyHelper.getSshPublicKeyString());

                    OpenSSHKeyFile key = new OpenSSHKeyFile();
                    key.init(new String(sshKeyHelper.getPEMRsaPrivateKey()), null);

                    ssh.authPublickey(sshUsername, key);
                    isAuthenticated = true;
                } catch (IOException e) {
                    if (warnIfNotYetReady)
                        test.warn("Exception while authenticating over SSH connection: " + e.getMessage(), e);
                    else
                        test.note("Exception while authenticating over SSH connection: " + e.getMessage(), e);
                    isAuthenticated = false;
                }
            if (isAuthenticated)
                break;

            test.note("Timing info: login attempt took " + (System.currentTimeMillis() - now) + " ms");

            if (warnIfNotYetReady)
                test.warn("SSH connection not successful (should be available from the moment that status is 'ready').");
            else
                test.note("SSH connection not successful (should be available from the moment that status is 'ready').");
            now = System.currentTimeMillis();
            if (now <= deadline)
                try {
                    long sleepTimeMs = 30000L;
                    if (deadline - now < sleepTimeMs) {
                        sleepTimeMs = deadline - now;

                        //give it enough room to be able to restart one more time
                        if (sleepTimeMs > 1000L)
                            sleepTimeMs -= 1000L;
                    }

                    test.note("Trying again in " + (sleepTimeMs / 1000) + " seconds...");

                    Thread.sleep(sleepTimeMs);
                } catch (InterruptedException e) { /* Ignore*/ }
            now = System.currentTimeMillis();
        }


        test.assertTrue(isAuthenticated, "Could not login to host: connection setup or authentication with private key failed. " +
                sshUsername + "@" + sshHostname + ":" + sshPort);

        test.note("SSH connection authenticated successfully after " + (System.currentTimeMillis() - start) + " milliseconds, will open session.");
        Session session = ssh.startSession();
        test.assertTrue(session != null);

        test.note("SSH session opened successfully, will send command.");
        //session.execCommand("/usr/bin/who");
        //the netstat command shows all SSH connections. This is useful to see if proxy is used correctly.
        String remoteCommand = "ls / && uname -a && who && (netstat -antW | grep ':22 ')";
        Session.Command command = session.exec(remoteCommand);
        command.join();
        BufferedReader sout = new BufferedReader(new InputStreamReader(command.getInputStream()));
        BufferedReader serr = new BufferedReader(new InputStreamReader(command.getErrorStream()));

        String result = "";
        String err = "";
        String line = sout.readLine();
        while (line != null) {
            result += line + "\n";
            line = sout.readLine();
        }
        line = serr.readLine();
        while (line != null) {
            err += line + "\n";
            line = serr.readLine();
        }
        //        note("\"who\" command on "+sshHostname+"@"+sshHostname+":"+sshPort+" result: \""+result.trim()+"\". (stderr is: \"" + err + "\")");
        //        assertTrue(result.contains(sshUsername), "I executed \"/usr/bin/who\" on the remote host, and expected a return containing my username (\"" + sshUsername + "\"). Instead I got: \"" + result.trim() + "\". (stderr is: \"" + err + "\")");

        test.note("\"" + remoteCommand + "\" command on " + sshUsername + "@" + sshHostname + ":" + sshPort + " \nresult: \"" + result.trim() + "\". \n(stderr is: \"" + err + "\")", true/*formatAsCode*/);
        test.assertTrue(result.length() > 5, "I executed \"" + remoteCommand + "\" on the remote host, and expected some reply. Instead I got: \"" + result.trim() + "\". (stderr is: \"" + err + "\")");

        session.close();
        ssh.close();
    }

    public SSHKeyHelper getSshKeyHelper() {
        return sshKeyHelper;
    }

    public String getSshUsername() {
        return sshUsername;
    }

    public String getSshHostname() {
        return sshHostname;
    }

    public int getSshPort() {
        return sshPort;
    }

    /**
     * Important! The SSH keypais is not stored in this SshProxyInfo object, because it is not known here.
     * The caller should fill it in. Typically, the caller does know the user SSH keys that need to be filled in here.
     *
     * @return the proxy set in the manifest. This proxy should be used when logging in using the sshUsername@sshHostname:sshPort login. May be null.
     */
    public JFedConnection.SshProxyInfo getManifestSshProxy() {
        return manifestSshProxy;
    }

    public JFedConnection.SshProxyInfo getSshProxy() {
        return sshProxy;
    }

    public void setNodeLoginForceIpVersion(int nodeLoginForceIpVersion) {
        this.nodeLoginForceIpVersion = nodeLoginForceIpVersion;
    }

    public String getPreferredUser() {
        return preferredUser;
    }


    public void checkManifestCorrectness(String rspec, TargetAuthority testedAuthority) {
        test.setErrorsAreWarnings(true);
        test.assertNotNull(rspec, "Manifest Rspec is null");

        //quick HACK to allow the planetlab Europe rspec manifest
        if (rspec.contains("<RSpec")) {
            test.note("We received an SFA RSpec instead of a GENI RSpec v3, but we'll try to work around.");
            //LOG.debug("HACK: rewriting received planetlab manifest XML to make it parsable xml");
            rspec =
                    rspec.replace("<RSpec type=\"SFA\"", "<rspec xmlns=\"http://www.geni.net/resources/rspec/3\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" type=\"manifest\"");
            rspec =
                    rspec.replace("<RSpec ", "<rspec xmlns=\"http://www.geni.net/resources/rspec/3\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" type=\"manifest\"");
            rspec = rspec.replace("</RSpec>", "</rspec>");
            rspec = rspec.replace("<network name=\"ple\">", "");
            rspec = rspec.replace("</network>", "");
            //LOG.debug("HACK: rewriten xml: "+rspec);
        }

        ModelRspec model = new ManifestRspecSource(rspec, ModelRspecType.BASIC).getModelRspec();
        test.assertNotNull(model, "Problem parsing manifest RSpec: manifest is empty or not valid!");

        List<? extends RspecNode> nodes = model.getNodes();
        test.assertNotEmpty(nodes, "There are no nodes in the manifest RSpec");

        boolean gotAuthNode = false;
        for (RspecNode node : nodes) {
            test.assertNotNull(node.getClientId(), "The manifest RSpec contained a node without a client_id. " +
                    "Explanation: The AM must keep the client_id of all nodes in the request RSpec intact. " +
                    "Clients need the client_id to know which request node matches which node in the manifest.");
            if (node.getClientId() != null) {
                test.assertNotNull(node.getComponentManagerId(), "The manifest RSpec contained a node without a component_manager_id. client_id=\"" + node.getClientId() + "\"");

                if (node.getComponentManagerId() != null && Objects.equals(node.getComponentManagerId(), testedAuthority.getRspecUrn())) {
                    test.assertNotNull(node.getSliverId(), "The manifest RSpec contained a node without a sliver_id. client_id=\"" + node.getClientId() + "\". " +
                            "Explanation: The sliver_id must be set to a sliver URN by the AM for each node for which it is a component_manager. " +
                            "Clients need the sliver_id to know which node in the request matches which sliver in the Status/SliverStatus calls reply.");
                    gotAuthNode = true;
                }
            }
        }
        test.assertTrue(gotAuthNode, "No nodes where found in the manifest for component_manager_id \"" + testedAuthority.getRspecUrn() + "\"");
        test.setErrorsAreWarnings(false);
    }
}
