package be.iminds.ilabt.jfed.lowlevel.api.test;

import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Server;
import be.iminds.ilabt.jfed.fedmon.webapi.service.json.Service;
import be.iminds.ilabt.jfed.log.Logger;
import be.iminds.ilabt.jfed.lowlevel.*;
import be.iminds.ilabt.jfed.lowlevel.api_wrapper.AggregateManagerWrapper;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnectionProvider;
import be.iminds.ilabt.jfed.lowlevel.connection.SfaConnection;
import be.iminds.ilabt.jfed.lowlevel.stitching.StitchingDirector;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.rspec.request.geni_rspec_3.*;
import be.iminds.ilabt.jfed.testing.base.ApiTest;
import be.iminds.ilabt.jfed.util.TargetAuthority;
import be.iminds.ilabt.jfed.util.TextUtil;
import net.schmizz.sshj.SSHClient;
import net.schmizz.sshj.connection.channel.direct.Session;
import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
import net.schmizz.sshj.userauth.UserAuthException;
import net.schmizz.sshj.userauth.keyprovider.OpenSSHKeyFile;
import org.slf4j.LoggerFactory;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.Unmarshaller;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.stream.Collectors;

/**
 * AbstractStitchingPingTest
 */
public class AbstractStitchingPingTest extends ApiTest {
    protected static final org.slf4j.Logger LOG = LoggerFactory.getLogger(StitchingPingTest.class);
    protected final Map<Server, NodeLoginTester> nodeLoginTesters = new HashMap<>();
    protected CommonAMTest.SliceInfo sliceInfo;
    protected String rspecXmlStr;
    protected Date expirationDate;
    protected String scsUrl;
    protected String scsUrn;
    protected List<String> stitchedAuthorityUrns;
    protected List<Server> stitchedAuthorities;
    protected Server scsServer;
    protected Boolean createSliceIfNotExists = true;
    protected Boolean reuseSliceIfExists = false;
    protected Boolean disableComplianceTests = false;
    protected Boolean disableCleanup = false;
    protected CommonAMTest commonAMTest;
    protected List<AnyCredential> userCredentials;
    protected final List<Node> nodes = new ArrayList<>();
    protected final List<Link> links = new ArrayList<>();

    protected StitchingDirector director;

    public AbstractStitchingPingTest(Logger logger, TargetAuthority testedAuthority, GeniUserProvider geniUserProvider, JFedConnectionProvider connectionProvider, TestbedInfoSource testbedInfoSource, JFedPreferences jFedPreferences) {
        super(logger, testedAuthority, geniUserProvider, connectionProvider, testbedInfoSource, jFedPreferences);
    }

    @Override
    public List<String> getReqConfKeys() {
        List<String> res = new ArrayList<>();
        res.add("stitchedAuthorityUrns"); //space seperated list
        return res;
    }

    @Override
    public List<String> getOptConfKeys() {
        List<String> res = new ArrayList<>();
        res.add("topology");

        CommonAMTest.addOptionsForFixedRspec(res);

        /* fixed_ssh_private_key_file and fixed_ssh_public_key_file option:
              There are 3 cases:
                  option not specified: use randomly generated key that is different each time
                  option specified, with private key file specified: use public and private key from files (public key can be read from PEM certificate)
                  option specified, with private key file specified, AND this file is the same file for both and is a user login file (key+cert file): use public and private key from file.
         */
        res.add("fixed_ssh_public_key_file"); //needed if fixed_ssh_private_key_file is specified (file containing PEM x509certificate is also allowed)
        res.add("fixed_ssh_private_key_file"); //needed if fixed_ssh_public_key_file is specified
        res.add("fixed_ssh_private_key_password"); //may be null,even if fixed_ssh_private_key_file is not (in case no password is needed)

        res.add("disable_slice_cleanup"); //no deleteSliver call

        //there are not much compliance tests here, so no need for this  res.add("disable_compliance_tests"); //test only for bare functionality, do not do extra checks

        res.add("create_slice_if_not_exists");
        res.add("reuse_slice_if_exists");
        res.add("fixed_slice_name"); //uses this slice name. If not specified, will generate a slice name automatically
        res.add("fixed_project_name"); //uses this as the project/subauthority. If not specified, will choose one if needed, or not use one otherwise.
        res.add("try_to_use_project_for_slices"); //if set and no project name set, this will try to find any project for the user, and use it to create slices.

        //fallback to default is not specified
        res.add("scsUrn");
        res.add("scsUrl");

        CommonAMTest.addOptionsForNodeLoginSshProxy(res);
        CommonAMTest.addOptionsForConnectionSshProxy(res);

        //speaksfor and delegation support
        CommonAMTest.addCredentialOptions(res);

        //future
//        res.add("minimum_slice_lifetime"); //minimum lifetime for slice
//        res.add("minimum_sliver_lifetime"); //minimum lifetime for slice

        return res;
    }

    @Override
    public String getTestDescription() {
        return "Test Stitching between multiple authorities, including node login and ping test between nodes. This version uses the AggregateManagerWrapper, and is thus not limited to AMv2. (and will actually prefer AMv3 if available.)";
    }

    protected AggregateManagerWrapper getAMWrapper(Server auth) {
        AggregateManagerWrapper amWrapper = getAutomaticAggregateManagerWrapperFactory().create(auth);

        //this has to be done after commonAMTest.processCredentialOptions in setUp()
        if (commonAMTest.getSpeaksForConfig() != null) {
            amWrapper.setSpeaksFor(commonAMTest.getSpeaksForConfig().getSpeaksForCredentials(), commonAMTest.getSpeaksForConfig().getUser2Urn());
            //Note: this isn't raelly needed ormally, as amWrapper.setSpeaksFor should do this.
            //  But it can be done manually, and we do so here, because the user might specify other options.
            if (!commonAMTest.getSpeaksForConfig().isAutomaticSpeaksForOptionNames())
                amWrapper.setExtraOptionsForCallsWithCredential(commonAMTest.addCredentialExtraOptions(null));
        }

        return amWrapper;
    }

    protected boolean isNonEmptyProperty(String key) {
        String val = getTestConfig().getProperty(key);
        if (val == null) return false;
        if (val.trim().isEmpty()) return false;
        return true;
    }

    @Override
    public void setUp() throws IOException, JFedException {
        stitchedAuthorityUrns = getStitchedAuthorityUrns();
        stitchedAuthorities = new ArrayList<>();
        for (String urn : stitchedAuthorityUrns) {
            Server auth = testbedInfoSource.getByUrnExact(urn);
            if (auth == null)
                errorFatal("Authority \"" + urn + "\" is not known.");
            stitchedAuthorities.add(auth);
        }

        commonAMTest = new CommonAMTest(this, user.getUserAuthorityServer(), stitchedAuthorities);

        int authIndex = 0;
        for (String urn : stitchedAuthorityUrns)
            commonAMTest.logParsableInfo("STITCH_AUTH" + (authIndex++) + "_URN", urn);

        boolean confHasScsUrl = isNonEmptyProperty("scsUrl");
        boolean confHasScsUrn = isNonEmptyProperty("scsUrn");

        if (!confHasScsUrl && !confHasScsUrn) {
            note("SCS config: no SCS urn or URL specified");
            Integer scsId = user.getUserAuthorityServer().getDefaultScsId();
            if (scsId != null) {
                Server scsServer = testbedInfoSource.getServerById(scsId);
                assert scsServer != null;
                Service scsService = ApiInfo.findService(scsServer, ApiInfo.ApiName.GENI_SCS, 1);

                if (scsService != null) {
                    scsUrn = scsService.getUrn();

                    URL defScsUrl = new URL(scsService.getUrl());
                    scsUrl = defScsUrl == null ? null : defScsUrl.toExternalForm();

                    note("No SCS configured: Defaulting to user SCS: scsUrn=" + scsUrn + "  scsUrl=" + scsUrl);
                    assertNotNull(scsUrn, "No default SCS known for user " + user.getUserUrn() + " at auth " + user.getUserAuthorityServer().getDefaultComponentManagerUrn());
                    assertNotNull(scsUrl, "No URL known for user (" + user.getUserUrn() + ") default SCS: " + scsUrn);
                } else {
                    note("SCS ID is known for the user's authority, BUT the actual SCS service details are not known. scsId=" + scsId);
                }
            } else {
                //no SCS known for user authority
                note("No SCS is known for the user's authority");
            }
        } else {
            if (confHasScsUrl)
                scsUrl = getTestConfig().getProperty("scsUrl").trim();
            else
                scsUrl = null;

            if (confHasScsUrn)
                scsUrn = getTestConfig().getProperty("scsUrn").trim();
            else
                scsUrn = null;

            note("SCS config: scsUrn=" + scsUrn + "  scsUrl=" + scsUrl);
        }

        assert testbedInfoSource != null;

        //create stitching auth if doesn't exist
        scsServer = null;

        if (scsUrn != null) {
            scsServer = testbedInfoSource.getByUrnExact(scsUrn);
        } else {
            if (scsUrl == null)
                fatalError("No SCS URN or SCS URL is known, and no default is known for user.");
            scsServer = testbedInfoSource.getServerByUrl(scsUrl);
        }

        //TODO: support this again? Better just add the SCS to the list...
//        if (scsServer == null && scsUrl != null) {
//            note("SCS unknown, will create it. Hopefully that works (since we won't trust it if it has a self signed SSL certificate).");
//            Map<ServerType, URL> urlMap = new HashMap<ServerType, URL>();
//            urlMap.put(new ApiInfo.Api(ApiInfo.ApiName.GENI_SCS, 1), new URL(scsUrl));
//            scsServer = new Server(scsUrn, "Stitching Test SCS", null, urlMap, null/*proxies*/, null, "scs");
//            testbedInfoSource.addAuthority(scsServer);
//            testbedInfoSource.fireChange();
//            assert testbedInfoSource.getByUrnExact(scsUrn) != null;
//        }

        if (scsServer == null)
            fatalError("Problem finding scsServer");

        if (scsUrl == null) {
            Service scsService = ApiInfo.findService(scsServer, ApiInfo.ApiName.GENI_SCS, 1);
            if (scsService == null)
                fatalError("SCS Server " + scsServer.getId() + " has no SCS Service");
            URL u = new URL(scsService.getUrl());
            if (u == null)
                fatalError("SCS Server " + scsServer.getId() + " has no SCS Service URL");
            scsUrl = u.toExternalForm();
        }
        if (scsUrn == null)
            scsUrn = scsServer.getDefaultComponentManagerUrn();

        note("Using SCS " + scsUrn + " @ " + scsUrl);
        commonAMTest.logParsableInfo("STITCH_SCS_URN", scsUrn);
        commonAMTest.logParsableInfo("STITCH_SCS_URL", scsUrl);


        //for backward compatibility, support "rspecfile" as if it was "fixed_rspec_file"
        String rspecFileArg = getTestConfig().getProperty("rspecfile");
        if (rspecFileArg != null && !rspecFileArg.isEmpty() && !getTestConfig().containsKey("fixed_rspec_file")) {
            getTestConfig().setProperty("fixed_rspec_file", rspecFileArg);
            note("The \"rspecfile\" option has been replaced by the \"fixed_rspec_file\" option. Update your config!");
        }
        String rspecArg = commonAMTest.getFixedRspecOption();

        if (rspecArg != null) {
            note("Will use a fixed_rspec as specified in the config, instead of automatically generating an RSpec.");
            if (!commonAMTest.isGeniFixedRspec())
                fatalError("The specified request RSpec is not a geni v3 RSpec. This test does not yet support that case.");
        }


        String topologyType = getTestConfig().getProperty("topology");
        if (topologyType == null || topologyType.isEmpty() || Objects.equals(topologyType, "ring") || Objects.equals(topologyType, "line") || Objects.equals(topologyType, "snake")) {
            //supported topology type
        } else
            errorFatal("Unknown topology of type \"" + topologyType + "\"");


        createSliceIfNotExists = TextUtil.objectToBoolean(getTestConfig().getProperty("create_slice_if_not_exists"));
        reuseSliceIfExists = TextUtil.objectToBoolean(getTestConfig().getProperty("reuse_slice_if_exists"));
        if (createSliceIfNotExists == null) createSliceIfNotExists = true;
        if (reuseSliceIfExists == null) reuseSliceIfExists = false;

        disableComplianceTests = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_compliance_tests"));
        disableCleanup = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_slice_cleanup"));
        if (disableComplianceTests == null) disableComplianceTests = false;
        if (disableCleanup == null) disableCleanup = false;

        expirationDate = new Date(System.currentTimeMillis() + (2 * 60 * 60 * 1000L)); //2 hours

        commonAMTest.processCredentialOptions(testbedInfoSource, authorityFinder); //handle speaksFor etc
    }

    public List<String> getStitchedAuthorityUrns() {
        List<String> res = new ArrayList<>();
        String urnList = getTestConfig().getProperty("stitchedAuthorityUrns");
        if (urnList == null)
            throw new RuntimeException("StitchingPingTest required config key stitchedAuthorityUrns is not present.");
        String[] urnsArray = urnList.split(" ");
        Collections.addAll(res, urnsArray);
        if (res.isEmpty())
            throw new RuntimeException("StitchingPingTest required config key stitchedAuthorityUrns (space seperated list) does not contain urns: " + urnList);
        if (res.size() == 1)
            throw new RuntimeException("StitchingPingTest required config key stitchedAuthorityUrns (space seperated list) contains only 1 urn: " + urnList);
        return res;
    }

    protected List<AnyCredential> getUserCredentialList() {
        assert userCredentials != null;
        assert !userCredentials.isEmpty();
        return userCredentials;
    }

    public SfaConnection getSAConnection() throws JFedException {
        return (SfaConnection) connectionProvider.getConnectionByAuthority(user, user.getUserAuthorityServer(), new ApiInfo.Api(ApiInfo.ApiName.PROTOGENI_SA, 1));
    }

    public SfaConnection getSCSConnection(Server scsAuth) throws JFedException {
        return (SfaConnection) connectionProvider.getConnectionByAuthority(user, scsAuth, new ApiInfo.Api(ApiInfo.ApiName.GENI_SCS, 1));
    }

    @Test()
    public void getUserCredentials() throws JFedException {
        userCredentials = commonAMTest.getUserCredentialListForAM();
    }

    @Test(hardDepends = {"getUserCredentials"}, groups = {"createslice"})
    public void createSlice() throws JFedException, IOException {
        //create a slice to use in AM tests
        String fixedSliceName = getTestConfig().getProperty("fixed_slice_name");
        String fixedProjectName =
                getTestConfig().getProperty("fixed_project_name"); //may be null, in that case, no sub authority is used, unless mandatory
        boolean tryToUseProject =
                TextUtil.objectToBoolean(getTestConfig().getProperty("try_to_use_project_for_slices")) != Boolean.FALSE; //so default is tru

        if (reuseSliceIfExists && fixedSliceName != null) {
            //check if exists: reuseSlice returns info if exists, null otherwise
            sliceInfo = commonAMTest.reuseSlice(fixedSliceName);

            if (sliceInfo != null)
                return;

            //else fall back to creating slice
            if (!createSliceIfNotExists)
                throw new RuntimeException("create_slice_if_not_exists is false, and slice does not exist.");
        }

        if (fixedSliceName == null)
            sliceInfo =
                    commonAMTest.createSlice("t", fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/); //TODO: support fixed_rspec option
        else
            sliceInfo =
                    commonAMTest.createSlice(fixedSliceName, false, fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/);

        assertNotNull(sliceInfo);
        assertNotNull(sliceInfo.urn);
        assertNotNull(sliceInfo.name);
        assertNotNull(sliceInfo.urnString);
        assertNotNull(sliceInfo.credentials);
        for (AnyCredential c : sliceInfo.credentials) {
            assertNotNull(c.getCredentialXml());
            assertTrue(c.getCredentialXml().length() > 10, "credential too short " + c.getCredentialXml());
        }
    }

    private List<String> allIps() {
        return nodes.stream()
                .flatMap(node -> node.ifaces.stream())
                .map(iface -> iface.ip)
                .collect(Collectors.toList());

    }

    private List<String> localIps(Server auth) {
        return nodes.stream()
                .filter(node -> Objects.equals(node.authUrn, auth.getDefaultComponentManagerUrn()))
                .flatMap(node -> node.ifaces.stream())
                .map(iface -> iface.ip).collect(Collectors.toList());
    }

    private List<String> pingTargetsForAuth(Server auth) {
        return nodes.stream()
                .filter(node -> Objects.equals(node.authUrn, auth.getDefaultComponentManagerUrn()))
                .flatMap(node -> node.ifaces.stream())
                .flatMap(iface -> iface.link.otherIfaces(iface).stream())
                .map(otherSideIface -> otherSideIface.ip)
                .collect(Collectors.toList());
    }

    /**
     * returns true if the authority is one of the authorities with nodes, and not just one of the stitching link auths
     */
    protected boolean isNodeAuth(Server auth) {
        return stitchedAuthorityUrns.contains(auth.getDefaultComponentManagerUrn());
    }

    private boolean isSiteNodeExclusive(String authUrn) {
//        Server auth = testbedInfoSource.getByUrn(authUrn);
//        assert auth != null;
//        boolean isExclusive = CommonAMTest.getExclusiveForSite(auth);

        if (authUrn.startsWith("urn:publicid:IDN+wall2.ilabt.iminds.be+authority+"))
            return true;
        return false;
    }

    private String getSiteComponentManagerId(String authUrn) {
        Server auth =
                testbedInfoSource.getByUrnExact(authUrn);
        assert auth != null;
        return CommonAMTest.getComponentManagerIdForSite(auth);
    }

    private String getSiteNodeSliverType(String authUrn) {
        Server auth =
                testbedInfoSource.getByUrnExact(authUrn);
        assert auth != null;
        String sliverType = CommonAMTest.getSliverTypeForSite(auth);

        if (authUrn.startsWith("urn:publicid:IDN+wall2.ilabt.iminds.be+authority+") ||
                authUrn.startsWith("urn:publicid:IDN+wall1.ilabt.iminds.be+authority+"))
            return "raw-pc";
        if (Objects.equals(sliverType, "raw-pc"))
            return "emulab-openvz";

        if (sliverType == null) //fallback. will probably fail
            return "emulab-openvz";

        return sliverType;
    }

    private void processFixedRspec(String inputRspec) {
        rspecXmlStr = inputRspec;
        try {
            JAXBContext jc =
                    JAXBContext.newInstance(be.iminds.ilabt.jfed.rspec.request.geni_rspec_3.RSpecContents.class);
            Unmarshaller u = jc.createUnmarshaller();
            //            JAXBElement<RSpecContents> doc = (JAXBElement<RSpecContents>) u.unmarshal( new FileInputStream(rspecFile));
            JAXBElement<be.iminds.ilabt.jfed.rspec.request.geni_rspec_3.RSpecContents> doc =
                    (JAXBElement<be.iminds.ilabt.jfed.rspec.request.geni_rspec_3.RSpecContents>) u.unmarshal(new StringReader(inputRspec));
            be.iminds.ilabt.jfed.rspec.request.geni_rspec_3.RSpecContents c = doc.getValue();

            assert c != null;

            //Note: getType() will return null if the type is specified but not "request".
            //      This is because the xsd specifies type as a choice from a list containing only "request".
            if (c != null &&
                    c.getType() != null &&
                    c.getType().value() != null) {
                if (!Objects.equals(c.getType().value(), "request"))
                    LOG.warn("Parsing an RSpec " + c.getType().value() + " as an RSpec request!");
            } else {
                LOG.warn("Parsed RSpec has no type (note: this also occurs if the type is not \"request\"): " + inputRspec);
            }

            Map<String, Iface> nameToIface = new HashMap<>();
            Map<String, Node> ifaceNameToRspecNode = new HashMap<>();

            //first nodes
            for (Object o : c.getAnyOrNodeOrLink()) {
                if (o instanceof JAXBElement) {
                    JAXBElement el = (JAXBElement) o;
                    if (el.getValue() instanceof NodeContents) {
                        NodeContents node = (NodeContents) el.getValue();

                        String compManagerId = node.getComponentManagerId();
                        String id = node.getClientId();
                        //in an advertisement, client ID is typically empty, but component name and ID are set
                        if (id == null) id = node.getComponentName();
                        if (id == null) id = node.getComponentId();

                        assert id != null : "Rspec has a node without any ID: " + el;
                        Node resNode = new Node(id, compManagerId);
                        nodes.add(resNode);

                        for (Object nodeElO : node.getAnyOrRelationOrLocation()) {
                            if (nodeElO instanceof JAXBElement) {
                                JAXBElement nodeEl = (JAXBElement) nodeElO;
                                if (nodeEl.getValue() instanceof InterfaceContents) {
                                    InterfaceContents ic = (InterfaceContents) nodeEl.getValue();
                                    assert ic != null;

                                    //for a request, this is mandatory, but we use the same code to parse the advertisement
                                    //assert ic.getClientId() != null : "InterfaceContents has no clientId: "+ic;
                                    if (ic.getClientId() != null) {
                                        //check if iface with same name is not already added
                                        for (String existingIfaceName : ifaceNameToRspecNode.keySet())
                                            if (Objects.equals(ic.getClientId(), existingIfaceName))
                                                throw new RuntimeException("Duplicate interfaces in XML Rspec: id=\"" + existingIfaceName + "\"");

                                        ifaceNameToRspecNode.put(ic.getClientId(), resNode);
                                    } else {
                                        LOG.warn("ignoring that InterfaceContents has no clientId");
                                    }
                                }
                            }
                        }
                    }
                }
            }

            //then the rest
            for (Object o : c.getAnyOrNodeOrLink()) {
                if (o instanceof JAXBElement) {
                    JAXBElement el = (JAXBElement) o;
                    boolean handled = false;
                    if (el.getValue() instanceof NodeContents)
                        handled = true;
                    if (el.getValue() instanceof LinkContents) {
                        handled = true;
                        LinkContents link = (LinkContents) el.getValue();

                        String id = link.getClientId();
                        //in an advertisement, client ID is typically empty, but component name and ID are set
                        if (id == null) id = link.getOtherAttributes().get("component_name");
                        if (id == null) id = link.getOtherAttributes().get("component_id");

                        assert id != null : "Rspec has a link without any ID: " + el;

                        List<String> linkComponentManagerUrns = new ArrayList<>();
                        for (Object oo : link.getAnyOrPropertyOrLinkType()) {
                            //sometimes, it's wrapped in a JAXBElement
                            Object value = oo;
                            if (o instanceof JAXBElement)
                                value = ((JAXBElement) oo).getValue();

                            if (value instanceof ComponentManager) {
                                ComponentManager cm = (ComponentManager) value;
                                linkComponentManagerUrns.add(cm.getName());
                            }
                        }

                        if (linkComponentManagerUrns.size() < 2) {
                            //link without stitching
                            continue;
                        }

                        Link resLink = new Link(); //iface creation will add links
                        links.add(resLink);

                        //check if all interfaces of link are known. We will ignore the link if they are not.
                        for (Object linkElO : link.getAnyOrPropertyOrLinkType()) {
                            if (linkElO instanceof JAXBElement) {
                                JAXBElement linkEl = (JAXBElement) linkElO;
                                if (linkEl.getValue() instanceof InterfaceRefContents) {
                                    InterfaceRefContents ic = (InterfaceRefContents) linkEl.getValue();

                                    Iface iface = nameToIface.get(ic.getClientId());
                                    if (iface == null) {
                                        Node rspecNode = ifaceNameToRspecNode.get(ic.getClientId());

                                        if (rspecNode == null) {
                                            LOG.warn("Interface has not been defined in a <node>, but is referred to in a <link>: " + ic.getClientId());
                                        } else
                                            iface =
                                                    new Iface(rspecNode, resLink); //constructor links Node and Link automatically
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } catch (AssertionError e) {
            LOG.warn("AssertionError reading Rspec XML (ping test will not be possible due to this): " + e.getMessage(), e);
            nodes.clear();
            links.clear();
        } catch (Exception e) {
            LOG.warn("Error reading Rspec XML (ping test will not be possible due to this): " + e.getMessage(), e);
            nodes.clear();
            links.clear();
        }
    }

    @Test(groups = {"init"})
    public void generateRspec() throws JFedException, IOException {
        String topologyType = getTestConfig().getProperty("topology");

        String rspecArg = commonAMTest.getFixedRspecOption();
        if (rspecArg != null) {
            if (!commonAMTest.isGeniFixedRspec())
                fatalError("The specified request RSpec is not a geni v3 RSpec. This test does not yet support that case.");
//            errorFatal("support for rspecfile parameter not yet implemented. \""+rspecFile+"\"");
//            return;

            processFixedRspec(rspecArg);

            Set<String> authUrns = nodes.stream().map(node -> node.authUrn).collect(Collectors.toSet());

            note("Parsed Rspec file. node count=" + nodes.size() + "   stitching link count=" + links.size());

            note("  Authority URNs=" + authUrns);
            stitchedAuthorityUrns = new ArrayList<>(authUrns);

            note("  Stitching IPs=" + allIps());

            return;
        }


        assert testbedInfoSource != null;
        assert stitchedAuthorityUrns != null;
        assert stitchedAuthorityUrns.size() > 1;

        note("creating topology between " + stitchedAuthorityUrns.size() + " component managers: " + stitchedAuthorityUrns);

        //create nodes
        for (String authUrn : stitchedAuthorityUrns) {
            Node n = new Node(authUrn);
        }

        //create link(s)
        assert nodes.size() > 1;
        if (topologyType == null || topologyType.isEmpty() || Objects.equals(topologyType, "ring")) {
            if (nodes.size() == 2) {
                note("making topology: simple 2 node connection");
                //create link if there are only 2 nodes
                new Link(nodes.get(0), nodes.get(1));
            } else {
                note("making topology: ring with " + nodes.size() + " nodes");
                //create links (ring topo)
                Node firstNode = nodes.get(0);
                Node lastNode = nodes.get(nodes.size() - 1);
                Node prevNode = lastNode;
                for (Node n : nodes) {
                    new Link(n, prevNode);
                    prevNode = n;
                }
            }
        } else {
            if (topologyType.equals("line") || topologyType.equals("snake")) {
                if (nodes.size() == 2) {
                    note("making topology: simple 2 node connection");
                    //create link if there are only 2 nodes
                    new Link(nodes.get(0), nodes.get(1));
                } else {
                    note("making topology: line with " + nodes.size() + " nodes");
                    //create links
                    Node prevNode = null;
                    for (Node n : nodes) {
                        if (prevNode != null)
                            new Link(n, prevNode);
                        prevNode = n;
                    }
                }
            } else {
                errorFatal("Unknown topology of type \"" + topologyType + "\"");
            }
        }

        //create Rspec
        String nodesXml = "";
        String linksXml = "";
        for (Node node : nodes) {
            String nodeXml =
                    "  <node client_id=\"" + node.name + "\" component_manager_id=\"" + getSiteComponentManagerId(node.authUrn) + "\" exclusive=\"" + (
                            isSiteNodeExclusive(node.authUrn) ? "true" : "false") + "\">\n" +
                            "    <sliver_type name=\"" + getSiteNodeSliverType((node.authUrn)) + "\"/>\n";
//                    "    <sliver_type name=\"emulab-openvz\"/>\n";

            for (Iface iface : node.ifaces)
                nodeXml += "    <interface client_id=\"" + iface.getName() + "\">\n" +
                        "      <ip address=\"" + iface.ip + "\" netmask=\"255.255.255.0\" type=\"ipv4\"/>\n" +
                        "    </interface>\n";

            nodeXml += "  </node>\n";
            nodesXml += nodeXml;
        }

        for (Link link : links) {
            String linkXml = "  <link client_id=\"" + link.name + "\">\n";
            for (String urn : link.componentManagerUrns())
                linkXml += "    <component_manager name=\"" + getSiteComponentManagerId(urn) + "\"/>\n";
            for (Iface iface : link.ifaces)
                linkXml += "    <interface_ref client_id=\"" + iface.getName() + "\"/>\n";


            for (int ifaceIndex1 = 0; ifaceIndex1 < link.ifaces.size(); ifaceIndex1++)
                for (int ifaceIndex2 = ifaceIndex1 + 1; ifaceIndex2 < link.ifaces.size(); ifaceIndex2++) {
                    Iface iface1 = link.ifaces.get(ifaceIndex1);
                    Iface iface2 = link.ifaces.get(ifaceIndex2);
                    //capacity is in demo rspec, and is probably needed:
                    //    I got an error with text "*** ERROR: vtopgen: Cannot mix trivial_ok|emulated with * bw" on emulab.net when not using it
                    //       (but that could also have other cause)
                    linkXml +=
                            "    <property source_id=\"" + iface1.getName() + "\" dest_id=\"" + iface2.getName() + "\" capacity=\"100000\"/>\n" +
                                    "    <property source_id=\"" + iface2.getName() + "\" dest_id=\"" + iface1.getName() + "\" capacity=\"100000\"/>\n";
//                    linkXml += "    <property source_id=\""+iface1.getName()+"\" dest_id=\""+iface2.getName()+"\"/>\n" +
//                               "    <property source_id=\""+iface2.getName()+"\" dest_id=\""+iface1.getName()+"\"/>\n";
                }
            linkXml += "  </link>\n";

            linksXml += linkXml;
        }

        rspecXmlStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
                "<rspec type=\"request\" \n" +
                "    xmlns=\"http://www.geni.net/resources/rspec/3\" \n" +
                "    xmlns:planetlab=\"http://www.planet-lab.org/resources/sfa/ext/planetlab/1\" \n" +
                "    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n" +
                "       xmlns:stitch=\"http://hpn.east.isi.edu/rspec/ext/stitch/0.1/\" \n" +
                "    xsi:schemaLocation=\"http://www.geni.net/resources/rspec/3 \n" +
                "    http://www.geni.net/resources/rspec/3/request.xsd\">  \n" +
                nodesXml +
                linksXml +
                "</rspec>";

        LOG.info("Generated Rspec: " + rspecXmlStr);
        note("Generated Rspec: " + rspecXmlStr, true/*formatAsCode*/);
    }

    @Test(hardDepends = {"createSlice", "generateRspec"}, groups = {"init"})
    public void initStitching() throws JFedException, NoSuchAlgorithmException, MalformedURLException {
        assert director != null;
    }

    @Test(hardDepends = {"overwrite to set correct deps"}, softDepends = {"describeAllResources"}, groups = {
            "nodelogin"})
    public void loginAndPing() throws JFedException, IOException {
        assert director != null;

        List<Server> sliverAuths = director.getActivelyInvolvedAuthorities();

        boolean allLoginOk = true;

        //only try login on authorities with nodes.
        List<Server> authsWithNode = sliverAuths.stream().filter(this::isNodeAuth).collect(Collectors.toList());

        assertTrue(authsWithNode.size() > 1, "Too few authorities with nodes (" + authsWithNode.size() + "): authsWithNode=" + authsWithNode + "   stitchedAuthorityUrns=" + stitchedAuthorityUrns);

        for (Server auth : authsWithNode) {
            NodeLoginTester nodeLoginTester = nodeLoginTesters.get(auth);
            if (nodeLoginTester == null) {
                errorNonFatal("No SSH login info was extracted for " + auth.getDefaultComponentManagerUrn());
                allLoginOk = false;
                continue;
            }
            LOG.info("Login to " + auth.getDefaultComponentManagerUrn() + "  " + nodeLoginTester);
            note("Login to " + auth.getDefaultComponentManagerUrn() + "  " + nodeLoginTester);

            if (nodeLoginTester.getSshHostname() == null) {
                errorNonFatal("No node / service / login in manifest RSpec, so SSH login cannot be tested for " + auth.getDefaultComponentManagerUrn());
                allLoginOk = false;
                continue;
            }

            //Test node login using SSH private key
            SSHClient ssh = new SSHClient();
            ssh.addHostKeyVerifier(new PromiscuousVerifier()); //don't verify host keys
            try {
                ssh.connect(nodeLoginTester.getSshHostname(), nodeLoginTester.getSshPort());
            } catch (IOException e) {
                errorNonFatal("Could not connect to node \"" + nodeLoginTester.getSshHostname() + "\" at port " + nodeLoginTester.getSshPort());
                allLoginOk = false;
                continue;
            }
            String privateKeyToPrint = new String(nodeLoginTester.getSshKeyHelper().getPEMPrivateKey());
            if (privateKeyToPrint.length() > 50) privateKeyToPrint = privateKeyToPrint.substring(0, 50);
            note("Trying to log in with PEM private key:\n" + privateKeyToPrint);

            OpenSSHKeyFile key = new OpenSSHKeyFile();
            key.init(new String(nodeLoginTester.getSshKeyHelper().getPEMRsaPrivateKey()), null);

            boolean isAuthenticated;
            try {
                ssh.authPublickey(nodeLoginTester.getSshUsername(), key);
                isAuthenticated = true;
            } catch (UserAuthException ex) {
                isAuthenticated = false;
            }

            setErrorsNotFatal();
            assertTrue(isAuthenticated, "Could not login to host. " + nodeLoginTester.getSshUsername() + "@" + nodeLoginTester.getSshHostname() + ":" + nodeLoginTester.getSshPort());
            setErrorsFatal();

            if (!isAuthenticated)
                continue;

            try (Session session = ssh.startSession()) {

                List<String> pingTargets = pingTargetsForAuth(auth);
                for (String targetIP : pingTargets) {
                    note("Trying to ping " + targetIP);
                /*
                Ping options on linux:
                       -c count
                              Stop after sending count ECHO_REQUEST packets. With deadline option, ping waits for count ECHO_REPLY packets, until the timeout expires.
                       -n     Numeric output only.  No attempt will be made to lookup symbolic names for host addresses.
                       -w deadline
                              Specify a timeout, in seconds, before ping exits regardless of how many packets have been sent or received. In this case ping does not stop after count packet are sent, it
                              waits either for deadline expire or until count probes are answered or for some error notification from network.
                       -W timeout
                              Time to wait for a response, in seconds. The option affects only timeout in absense of any responses, otherwise ping waits for two RTTs.
                * */

                    String command = "ping -c 5 -n -w 30 " + targetIP;
                    String result = "";
                    String err = "";
                    try {
                        Session.Command comm = session.exec(command);

                        BufferedReader sout = new BufferedReader(new InputStreamReader(comm.getInputStream()));
                        BufferedReader serr = new BufferedReader(new InputStreamReader(comm.getErrorStream()));

                        String line = sout.readLine();
                        while (line != null) {
                            result += line + "\n";
                            line = sout.readLine();
                        }
                        line = serr.readLine();
                        while (line != null) {
                            err += line + "\n";
                            line = serr.readLine();
                        }
                    } finally {
                        session.close();
                    }
                    note("\"" + "ping -c 5 -n -w 30 " + targetIP + "\" command on " + nodeLoginTester.getSshUsername() + "@" + nodeLoginTester.getSshHostname() + ":" + nodeLoginTester.getSshPort() + " result: \"" + result.trim() + "\". (stderr is: \"" + err + "\")");
                    setErrorsNotFatal();
                    assertTrue(result.length() > 5, "I executed \"ping\" on the remote host, and expected some reply. Instead I got: \"" + result.trim() + "\". (stderr is: \"" + err + "\")");

                    commonAMTest.checkPingResult(session, result);
                    setErrorsFatal();
                }

            }
        }

        assertTrue(allLoginOk, "one or more login tests failed");
    }

    @Test(softDepends = {"loginAndPing", "overwrite to set dep tests!"}, hardDepends = {
            "createSlice"}, groups = {"createsliver", "nodelogin"})
    public void callDeletes() throws JFedException {
        assert director != null;

        if (disableCleanup) skip("No cleanup needed.");

        List<Server> allAuths = director.getActivelyInvolvedAuthorities();

        if (allAuths.size() == 0) {
            warn("Something went wrong: got empty list of involved authorities! This might be correct, but it could be a bug. Will instead Delete on all authorities to be sure everything is cleaned up.");
            allAuths = director.getAllAuthorities();
        }

        for (Server authorityInfo : allAuths) {
            //do delete call
            try {
                AggregateManagerWrapper amWrapper = getAMWrapper(authorityInfo);
                amWrapper.deleteSliver(sliceInfo.urn, sliceInfo.credentials);
                note("Called delete on " + authorityInfo.getDefaultComponentManagerUrn());
            } catch (JFedException e) {
                warn("Error while calling Delete on " + authorityInfo.getDefaultComponentManagerUrn() + ". Will continue deleting.");
            }
        }
    }

    private class Node {
        private final String name;
        private final String authUrn;
        private final List<AbstractStitchingPingTest.Iface> ifaces = new ArrayList<>();

        private Node(String authUrn) {
            this.name = "PC" + nodes.size();
            this.authUrn = authUrn;
            nodes.add(this);
        }

        private Node(String name, String authUrn) {
            this.name = name;
            this.authUrn = authUrn;
            nodes.add(this);
        }
    }

    private class Iface {
        private final Node node;
        private final AbstractStitchingPingTest.Link link;
        private final String ip;

        private Iface(Node node, AbstractStitchingPingTest.Link link) {
            this.node = node;
            this.link = link;
            this.node.ifaces.add(this);
            this.link.ifaces.add(this);
            this.ip = link.subnet + "." + (link.ifaces.indexOf(this) + 1);
        }

        private String getName() {
            return node.name + ":if" + node.ifaces.indexOf(this);
        }
    }

    private class Link {
        private final String name;
        private final String subnet;
        private final List<Iface> ifaces = new ArrayList<>();

        private Link() {
            this.name = "link" + links.size();
            this.subnet = "192.168." + (4 + links.size());
            links.add(this);
        }

        private Link(Node a, Node b) {
            this();
            new Iface(a, this);
            new Iface(b, this);
        }

        private List<String> componentManagerUrns() {
            return ifaces.stream().map(i -> i.node.authUrn).collect(Collectors.toList());
        }

        private List<Iface> otherIfaces(Iface iface) {
            return ifaces.stream().filter(i -> i != iface).collect(Collectors.toList());
        }
    }
}
