package be.iminds.ilabt.jfed.lowlevel.api.test;

import be.iminds.ilabt.jfed.log.Logger;
import be.iminds.ilabt.jfed.lowlevel.*;
import be.iminds.ilabt.jfed.lowlevel.api.AbstractGeniAggregateManager;
import be.iminds.ilabt.jfed.lowlevel.api.AggregateManager2;
import be.iminds.ilabt.jfed.lowlevel.api.user_spec.UserSpec;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnectionProvider;
import be.iminds.ilabt.jfed.lowlevel.connection.SfaConnection;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.NodeContents;
import be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.RSpecContents;
import be.iminds.ilabt.jfed.testing.base.ApiTest;
import be.iminds.ilabt.jfed.util.GeniUrn;
import be.iminds.ilabt.jfed.util.TargetAuthority;
import be.iminds.ilabt.jfed.util.TextUtil;

import javax.inject.Inject;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import java.io.IOException;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.util.*;

/**
 * TestAggregateManager2
 */
public class TestAggregateManager2 extends ApiTest {
    protected CommonAMTest commonAMTest;
    protected AggregateManager2 am2;
    protected List<AnyCredential> userCredentials;
    protected NodeLoginTester nodeLoginTester;
    protected Boolean createSliceIfNotExists = true;
    protected Boolean reuseSliceIfExists = false;
    protected Boolean beLessStrict = false;
    protected Boolean disableComplianceTests = false;
    protected Boolean disableCleanup = false;
    protected Map versionRawResult = null;
    protected AggregateManager2.VersionInfo versionInfo = null;

    protected CommonAMTest.SliceInfo sliceNoSliver;
    //helpers for finding a fixed node in the list
    protected String fixedNodeUrn = null;
    protected String fixedNodeSliverType = null;
    protected CommonAMTest.SliceInfo sliceSliver;
    protected String createSliverManifestRspec = null;

    @Inject
    public TestAggregateManager2(Logger logger, TargetAuthority testedAuthority, GeniUserProvider geniUserProvider, JFedConnectionProvider connectionProvider, TestbedInfoSource testbedInfoSource, JFedPreferences jFedPreferences) {
        super(logger, testedAuthority, geniUserProvider, connectionProvider, testbedInfoSource, jFedPreferences);
    }

    public String getTestDescription() {
        return "Many Aggregate Manager (Geni AM API v2) Tests. 2 slices and a sliver will be created during the tests. " +
                "The sliver will be deleted. " +
                "This will not test ListResources when requesting an advertisement request " +
                "(takes too long, use TestAggregateManager2AllListResources).";
    }

    @Override
    public List<String> getOptConfKeys() {
        List<String> res = new ArrayList<>();
        /* fixed_ssh_private_key_file and fixed_ssh_public_key_file option:
              There are 3 cases:
                  option not specified: use randomly generated key that is different each time
                  option specified, with private key file specified: use public and private key from files (public key can be read from PEM certificate)
                  option specified, with private key file specified, AND this file is the same file for both and is a user login file (key+cert file): use public and private key from file.

         */
        res.add("fixed_ssh_public_key_file"); //needed if fixed_ssh_private_key_file is specified (file containing PEM x509certificate is also allowed)
        res.add("fixed_ssh_private_key_file"); //needed if fixed_ssh_public_key_file is specified
        res.add("fixed_ssh_private_key_password"); //may be null,even if fixed_ssh_private_key_file is not (in case no password is needed)

        res.add("skip_final_manifest_check"); //don't check the manifest too much

        res.add("nodelogin_deadline_ms");

        res.add("create_slice_if_not_exists");
        res.add("reuse_slice_if_exists");
        res.add("fixed_slice_name"); //uses this slice name. If not specified, will generate a slice name automatically
        res.add("fixed_project_name"); //uses this as the project/subauthority. If not specified, will choose one if needed, or not use one otherwise.
        res.add("try_to_use_project_for_slices"); //if set and no project name set, this will try to find any project for the user, and use it to create slices.
        res.add("be_less_strict"); //turns some failures into warnings
        res.add("disable_compliance_tests"); //test only for bare functionality, do not do extra checks
        res.add("disable_slice_cleanup"); //no deleteSliver call
        CommonAMTest.addOptionsForFixedRspec(res);

        //speaksfor and delegation support
        CommonAMTest.addCredentialOptions(res);

        CommonAMTest.addOptionsForNodeLoginSshProxy(res);
        CommonAMTest.addOptionsForConnectionSshProxy(res);

        CommonAMTest.addOptionsForRspecTypeVersion(res);

        return res;
    }

    protected List<AnyCredential> getUserCredentialList() {
        return commonAMTest.getUserCredentialListForAM();
    }

    public SfaConnection getAM2Connection() throws JFedException {
        return (SfaConnection) connectionProvider.getConnectionByAuthority(
                user,
                testedAuthority.getServerToConnect(),
                new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 2));
    }

    protected void assertValidStatus(String status) {
        List<String> apiPossibleStatus = new ArrayList<>();
        apiPossibleStatus.add("ready");
        apiPossibleStatus.add("configuring");
        apiPossibleStatus.add("failed");
        apiPossibleStatus.add("unknown");

        List<String> otherPossibleStatus = new ArrayList<>();
        //not valid! but it does occur a lot in practice, so we will not fail but warn for it
        otherPossibleStatus.add("changing");
        otherPossibleStatus.add("notready");

        List<String> possibleStatus = new ArrayList<>();
        possibleStatus.addAll(apiPossibleStatus);
        possibleStatus.addAll(otherPossibleStatus);

        assertTrue(possibleStatus.contains(status),
                "Invalid geni_status: \"" + status + "\" is not one of " + possibleStatus);

        if (!apiPossibleStatus.contains(status))
            warn("SliverStatus returns invalid geni_status: \"" + status + "\" is not one of " + apiPossibleStatus);
    }

    /**
     * Check for correctness of a AM2 XmlRpc result. Should be tested for each reply received.
     */
    public void testAM2CorrectnessXmlRpcResult(Map res) {
        assertNotNull(res, "testAM2CorrectnessXmlRpcResult res is null");
        Object code = res.get("code");
        Object value = res.get("value");
        Object output = res.get("output");
        assertNotNull(code, "testAM2CorrectnessXmlRpcResult code == null in " + res);
        assertNotNull(value, "testAM2CorrectnessXmlRpcResult value == null in " + res);
        assertInstanceOf(code, Map.class, "testAM2CorrectnessXmlRpcResult code is not Map in " + res);
        Map codeHt = (Map) code;
        Object genicode = codeHt.get("geni_code");
        assertNotNull(genicode, "testAM2CorrectnessXmlRpcResult code does not contain \"geni_code\" in " + res);
        assertEquals(genicode.getClass(), Integer.class, "testAM2CorrectnessXmlRpcResult code.geni_code is not int in " + res);

        int genicodevalue = (Integer) genicode;
        //output should be present if code is not 0
        if (genicodevalue != 0) {
            assertNotNull(output, "testAM2CorrectnessXmlRpcResult: while geni_code is non success (" + genicodevalue + "), output == null in " + res);
            assertEquals(output.getClass(), String.class, "testAM2CorrectnessXmlRpcResult: while geni_code is non success (" + genicodevalue + "), output is not String (it is " + output.getClass().getName() + " with value \"" + output.toString() + "\") in \"+res+\"");
        } else {
            if (output == null)
                note("testAM2CorrectnessXmlRpcResult: while geni_code is success (" + genicodevalue + "), output is ommited. This is allowed by the API.");
            if (output != null && !(output instanceof String))
                warn("testAM2CorrectnessXmlRpcResult: while geni_code is success (" + genicodevalue + "), output is not String (it is " + output.getClass().getName() + " with value \"" + output.toString() + "\"). This is allowed but not recommended by jFed.");
        }
    }

    public void setUp() {
        //commonAMTest can create slices on demand and get user credential
        commonAMTest = new CommonAMTest(this);
        am2 = new AggregateManager2(logger, jFedPreferences);

        commonAMTest.processCredentialOptions(testbedInfoSource, authorityFinder); //handle speaksFor etc

        //user credential needs to be retrieved from SA for most tests
        System.out.println("Fetching User credential needed for AM tests");
        userCredentials = commonAMTest.getUserCredentialListForAM();

        createSliceIfNotExists = TextUtil.objectToBoolean(getTestConfig().getProperty("create_slice_if_not_exists"));
        reuseSliceIfExists = TextUtil.objectToBoolean(getTestConfig().getProperty("reuse_slice_if_exists"));
        disableComplianceTests = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_compliance_tests"));
        disableCleanup = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_slice_cleanup"));
        if (createSliceIfNotExists == null) createSliceIfNotExists = true;
        if (reuseSliceIfExists == null) reuseSliceIfExists = false;
        if (disableComplianceTests == null) disableComplianceTests = false;
        if (disableCleanup == null) disableCleanup = false;

        if (getTestConfig().get("be_less_strict") != null) {
            Object v = getTestConfig().get("be_less_strict");
            note("be_less_strict option specified: " + v);
            beLessStrict = TextUtil.objectToBoolean(v + "");
            if (beLessStrict)
                note(" be_less_strict activated: turning some failures into warnings");
            else
                note(" be_less_strict is NOT activated");
        }

        commonAMTest.setupProxyForConnectionPool(getTestConfig());
    }

    AggregateManager2.AggregateManagerReply<AggregateManager2.VersionInfo> getVersionReply;

    @Test(groups = {"getversion", "nonodelogin"})
    public void testGetVersionXmlRpcCorrectness() throws JFedException {
        getVersionReply = am2.getVersion(getAM2Connection());

        testAM2CorrectnessXmlRpcResult(getVersionReply.getRawResult());

        //GetVersion is the only method where we have to check for something extra in the raw XMLRPC result
        versionRawResult = getVersionReply.getRawResult();
        assertNotNull(versionRawResult);

        Object ver =
                versionRawResult.get("geni_api"); //this is here for backward compatibility with v1. GetVersion returns this on all versions.
        assertTrue(ver != null);
        assertTrue(ver instanceof Integer);
        int v = (Integer) ver;
        assertTrue(v == 2);

        assertEquals(getVersionReply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "GeniResponse code is not SUCCESS (0)");

        versionInfo = getVersionReply.getValue();
        assertNotNull(versionInfo);
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, groups = {"nonodelogin"})
    public void testGetVersionResultCorrectness() throws JFedException {
        Map value = (Map) versionRawResult.get("value");

        //check if all required fields are present
        assertEquals(value.get("geni_api"), 2);

        assertNotNull(value.get("geni_api_versions"), "geni_api_versions not in version result");
        assertTrue(value.get("geni_api_versions") instanceof Map, "geni_api_versions is not a Map but a " + value.get("geni_api_versions").getClass().getName());

        String[] rspecVersionListnames = {"geni_request_rspec_versions", "geni_ad_rspec_versions"};
        for (String name : rspecVersionListnames) {
            assertNotNull(value.get(name), name + " not in version result");


            Object[] v =  assertInstanceOf(value.get(name), Object[].class, name + " is not a List but a " + value.get(name).getClass().getName());
            assertTrue(v.length > 0, name + " array is empty");
            for (Object o : v) {
                assertTrue(o instanceof Map, name + " array should contain only Map not a " + o.getClass().getName());
                Map t = (Map) o;
                assertMapContainsNonemptyString(t, "type");
                assertMapContainsNonemptyString(t, "version");
                assertMapContainsString(t, "schema");
                assertMapContainsString(t, "namespace");
                Object extensionsO = t.get("extensions");
                assertNotNull(extensionsO);
                Object[] extensions = assertInstanceOf(extensionsO, Object[].class, "value for extensions of " + name + " is not a String but a " + extensionsO.getClass().getName());
                for (Object e : extensions)
                    assertTrue(e instanceof String, "an extension of \"+name+\" is not a string but a " + e.getClass().getName());
            }
        }
    }

    @Test(hardDepends = {"testGetVersionResultCorrectness"}, groups = {"nonodelogin"})
    public void testGetVersionResultApiVersionsCorrectness() throws JFedException, MalformedURLException {
        Map value = (Map) versionRawResult.get("value");
        Map versions = (Map) value.get("geni_api_versions");

        //test if contains self reference
        Object o = versions.get(2);
        if (o == null) {
            note("geni_api_versions for Integer 2 is null. (Note that for String \"2\" it is \"" + versions.get("2") + "). This test will accept String instead of int: the API specifies it should be int, but almost every server uses a String.");
            o = versions.get("2");
        }

        assertNotNull(o, "geni_api_versions for value 2 is null (tried with both int 2 and string \"2\").");
        assertTrue(o instanceof String, "value for 2 is not a String but a " + o.getClass().getName());

        boolean serverUrlWithLocalhost = false;
        try {
//            assertNotNull(getVersionReply);
            URL serverUrl = new URL(getVersionReply.getXMLRPCCallDetails().getServerUrl());
            serverUrlWithLocalhost = serverUrl.getHost().equals("localhost") || serverUrl.getHost().equals("127.0.0.1");
        } catch (Exception e) {
            note("Unexpected exception in test. Test will ignore this, but this might be an automated tester bug", e);
        }

        if (!serverUrlWithLocalhost) {
            //test that no url's are correct and none have localhost as hostname
            for (Object key : versions.keySet()) {
                assertTrue(key instanceof Integer || key instanceof String, "geni_api_versions keys should be Integer or String, not " + key.getClass().getName());
                int versionNr;
                if (key instanceof Integer)
                    versionNr = (Integer) key;
                else {
                    note("geni_api_versions contains String key \"" + key + "\". This test will accept String instead of int: the API specifies it should be int, but almost every server uses a String.");
                    versionNr = Integer.parseInt((String) key);
                }
                Object val = versions.get(key);
                assertTrue(val instanceof String, "geni_api_versions values should be String, not " + val.getClass().getName());
                String urlS = (String) val;
                //check URL
                URL url = new URL(urlS);
                String host = url.getHost();
                assertFalse(Objects.equals(host, "localhost"), "Illegal host in URL: " + url + " (host in URL should not be the non-global \"" + host + "\")");
                assertFalse(Objects.equals(host, "127.0.0.1"), "Illegal host in URL: " + url + " (host in URL should not be the non-global \"" + host + "\")");
            }
        } else
            note("The URL of the server has localhost. This test will assume that this is a test server, and will therefor NOT warn or fail if the URL's in GetVersion have localhost in them.");

        Object urn = value.get("urn");
        if (urn != null) {
            setErrorsNotFatal();
            assertInstanceOf(urn, String.class, "if URN is defined, it should be a String, not a " + urn.getClass().getName());
            if (urn instanceof String) {
                String urnString = (String) urn;
                String testUrnString = testedAuthority.getServerToConnect().getDefaultComponentManagerUrn();
                if (!Objects.equals(urnString, testUrnString))
                    warn("URN in GetVersion (" + urnString + ") does not match the URN used to start this test (" + testUrnString +
                            "). This could either be a problem on the server, a bad URN in GetVersion, or it could be that " +
                            "the jFed has wrong info about this authority. This failure could result in a failure to allocate " +
                            "resources (as the component_manager urn used will not match this server).");
            }
            setErrorsNotFatal();
        } else {
            note("GetVersion does not specify \"urn\" (which contains the component manager urn). This is not mandatory, but it is useful to include.");
        }
    }

    @Test(hardDepends = {"testGetVersionResultCorrectness"}, softDepends = {
            "testGetVersionResultApiVersionsCorrectness"}, groups = {"nonodelogin"})
    public void testGetVersionResultNoDuplicates() throws JFedException {
        //Check if RSpecs are unique
        List<AggregateManager2.VersionInfo.RspecVersion> adRspecVersions =
                new ArrayList<>();
        for (AggregateManager2.VersionInfo.RspecVersion rspecVer : versionInfo.getAdRspecVersions()) {
            for (AggregateManager2.VersionInfo.RspecVersion other : adRspecVersions)
                assertFalse(other.equalTypeAndVersion(rspecVer), "VersionInfo Result invalid: Duplicate Rspec type/version pair in supported Advertisement RSpec:" +
                        "type=" + other.getType() + " version=" + other.getVersion() + " VS " +
                        "type=" + rspecVer.getType() + " version=" + rspecVer.getVersion() + " ");
            adRspecVersions.add(rspecVer);
        }

        List<AggregateManager2.VersionInfo.RspecVersion> reqRspecVersions =
                new ArrayList<>();
        for (AggregateManager2.VersionInfo.RspecVersion rspecVer : versionInfo.getRequestRspecVersions()) {
            for (AggregateManager2.VersionInfo.RspecVersion other : reqRspecVersions)
                assertFalse(other.equalTypeAndVersion(rspecVer), "VersionInfo Result invalid: Duplicate Rspec type/version pair in supported Request RSpec" +
                        "type=" + other.getType() + " version=" + other.getVersion() + " VS " +
                        "type=" + rspecVer.getType() + " version=" + rspecVer.getVersion() + " ");
            reqRspecVersions.add(rspecVer);
        }
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultNoDuplicates"}, groups = {"nonodelogin"})
    public void testListResourcesBadCredential() throws JFedException {
        //Test without credentials. Should fail.
        AggregateManager2.AggregateManagerReply<String> reply = am2.listResources(
                getAM2Connection(),
                new ArrayList<>(),
                commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()),
                true,
                true,
                null,
                null);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertFalse(reply.getGeniResponseCode().isSuccess());
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultNoDuplicates"}, groups = {"nonodelogin"})
    public void testSliverStatusBadSlice() throws JFedException {
        String badSliceUrn =
                "urn:publicid:IDN+" + user.getUserAuthorityServer().getUrnTld() + "+slice+NonExisting";

        AggregateManager2.AggregateManagerReply<AggregateManager2.SliverStatus> reply =
                am2.sliverStatus(getAM2Connection(), getUserCredentialList(), badSliceUrn, commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertFalse(reply.getGeniResponseCode().isSuccess(), "SliverStatus reply GeniResponse code is SUCCESS (0) when given a non existing slice \"" + badSliceUrn + "\"");
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultNoDuplicates"}, groups = {"nonodelogin"})
    public void testCreateSliceNoSliver() throws JFedException, IOException {
        //create a slice to use in AM tests

        String fixedProjectName =
                getTestConfig().getProperty("fixed_project_name"); //may be null, in that case, no sub authority is used, unless mandatory
        boolean tryToUseProject =
                TextUtil.objectToBoolean(getTestConfig().getProperty("try_to_use_project_for_slices")) != Boolean.FALSE; //so default is tru
        sliceNoSliver =
                commonAMTest.createSlice("n", fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/); //not actually used with any rspec
        assert sliceNoSliver != null;
        assert sliceNoSliver.urn != null : "sliceNoSliver.urn is null";
        assert sliceNoSliver.urn.getValue().startsWith("urn:publicid:IDN+");
        assert sliceNoSliver.credentials != null : "sliceNoSliver.credential is null";
        assert !sliceNoSliver.credentials.isEmpty() : "sliceNoSliver.credential is empty";
    }

    @Test(hardDepends = {"testCreateSliceNoSliver"}, groups = {"nosliver", "nonodelogin"})
    public void testSliverStatusNoSliverSlice() throws JFedException {
        assert sliceNoSliver != null;

        //then check if SliverStatus reports not found as it should
        AggregateManager2.AggregateManagerReply<AggregateManager2.SliverStatus> reply =
                am2.sliverStatus(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceNoSliver), sliceNoSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertFalse(reply.getGeniResponseCode().isSuccess(),
                "SliverStatus reply GeniResponse code is SUCCESS (0) when given a slice \"" + sliceNoSliver.urn + "\" that has no sliver. (should fail because no sliver found for the slice)");

        assertNotNull(reply.getRawResult());
        Object rawValueResult = reply.getRawResult().get("value");
        if (rawValueResult != null) {
            //it does not need to have a result, but when it does, it should be a valid one.
            warnIfNot(rawValueResult instanceof Map, "SliverStatus result is not null, which is allowed on failure," +
                    " but in that case it is expected to be a valid response, which would be a struct. However, it is a " +
                    rawValueResult.getClass().getSimpleName() + " -> value=\"" + rawValueResult.toString() + "\"");
        }
        warnIfNot((rawValueResult != null) == (reply.getValue() != null),
                "Mismatch: SliverStatus processed result value is " +
                        (reply.getValue() != null ? "not null" : "null") + " but RAW SliverStatus result value is " +
                        (rawValueResult != null ? "not null" : "null")
        );
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultNoDuplicates"}, groups = {"listavailableresources", "nodelogin", "nonodelogin"})
    public void testListResourcesAvailableNoSlice() throws JFedException {
        String rspec =
                testListResourcesAvailableNoSlice_internal(commonAMTest.getRspecTypeName(getTestConfig()).toLowerCase(), commonAMTest.getRspecTypeVersion(getTestConfig()), true);

        String testUrnString = testedAuthority.getRspecUrn().getValue();

        //check if it contains any nodes component_manager_urn matching the testUrnString
        int nodecount = 0;
        int matchingNodecount = 0;
        Set<String> componentManagerIds = new HashSet<>();

        //helpers for finding a fixed node in the list
        String fixedNodeUrn = null;
        String fixedNodeSliverType = null;

        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \"" + commonAMTest.getRspecTypeName(getTestConfig()) + "\" \"" + commonAMTest.getRspecTypeVersion(getTestConfig()) + "\". " +
                    "This test does not support this, and will not check the validity of the result.");
            return;
        }

        try {
            Class docClass = be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.RSpecContents.class;
            String packageName = docClass.getPackage().getName();
            JAXBContext jc = JAXBContext.newInstance(packageName);
            Unmarshaller u = jc.createUnmarshaller();
            JAXBElement<RSpecContents> doc =
                    (JAXBElement<be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.RSpecContents>) u.unmarshal(new StringReader(rspec));
            be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.RSpecContents c = doc.getValue();

            String typ = c.getType();
            assertNotNull(typ, "Received advertisement RSpec does not specify a type: " + rspec);
            assertEquals(typ, "advertisement", "Received advertisement RSpec is not a advertisement, it is a: \"" + typ + "\"");

            for (Object o : c.getAnyOrNodeOrLink()) {
                if (o instanceof JAXBElement) {
                    JAXBElement el = (JAXBElement) o;
                    if (el.getValue() instanceof be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.NodeContents) {
                        be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.NodeContents node =
                                (be.iminds.ilabt.jfed.rspec.advertisement.geni_rspec_3.NodeContents) el.getValue();

                        nodecount++;

                        String componentManagerId = node.getComponentManagerId();
                        String nodeId = node.getComponentId();
                        componentManagerIds.add(componentManagerId);

                        if (Objects.equals(componentManagerId, testUrnString))
                            matchingNodecount++;

                        for (Object nodeElO : node.getAnyOrRelationOrLocation()) {
                            if (nodeElO instanceof JAXBElement) {
                                JAXBElement nodeEl = (JAXBElement) nodeElO;
                                if (nodeEl.getValue() instanceof NodeContents.SliverType) {
                                    NodeContents.SliverType st = (NodeContents.SliverType) nodeEl.getValue();
                                    String sliverTypeName = st.getName();

                                    if (Objects.equals(componentManagerId, testUrnString)) {
                                        if (fixedNodeUrn == null ||
                                                CommonAMTest.preferSliverType(sliverTypeName, fixedNodeSliverType)) {
                                            fixedNodeUrn = nodeId;
                                            fixedNodeSliverType = sliverTypeName;
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } catch (JAXBException e) {
            throw new RuntimeException("Exception parsing advertisement RSpec Xml: " + e.getMessage(), e);
        }

        if (matchingNodecount == 0) {
            if (componentManagerIds.isEmpty())
                note("Could not find a single node for this component_managers_id (" + testUrnString + ") in the advertisement. No nodes of other component_managers_id's found either.");
            else
                warn("Could not find a single node for this component_managers_id (" + testUrnString + ") in the advertisement. Did find nodes for the following component_managers_id's: " + componentManagerIds);
            note("There are probably no free nodes left");
        } else {
            note("Found nodes for the following component_managers_id's: " + componentManagerIds);
        }

        if (fixedNodeUrn == null)
            note("Could not find a suitable fixed node in the advertisement request");
        else
            note("Found potential fixed node: " + fixedNodeUrn);
        note("Best sliver type found: " + fixedNodeSliverType);
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {"testGetVersionResultNoDuplicates",
                                                                            "testListResourcesAvailableNoSlice"}, groups = {
            "listavailableresources", "nonodelogin"})
    public void testListResourcesAvailableNoSliceUpperCase() throws JFedException {
        note("This tests if uppercase rspec type also works. The AMv2 API says: \"type and version are case-insensitive strings, matching those in geni_ad_rspec_versions as returned by GetVersion at this aggregate.\"");
        testListResourcesAvailableNoSlice_internal(commonAMTest.getRspecTypeName(getTestConfig()).toUpperCase(), commonAMTest.getRspecTypeVersion(getTestConfig()), true);
    }

    public String testListResourcesAvailableNoSlice_internal(String rspecType, String rspecVersion, boolean available) throws JFedException {
        assert userCredentials != null;
        assert !userCredentials.isEmpty();

        //check if SliverStatus reports not found as it should
        //check without available option
        AggregateManager2.AggregateManagerReply<String> reply =
                am2.listResources(getAM2Connection(), getUserCredentialList(), rspecType, rspecVersion, available/*available*/, true/*compressed*/, null, commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given no slice urn and available=" + available);

        String rspec = reply.getValue();
        assertNotNull(rspec, "ListResources reply is completely empty when given no slice urn and available=" + available);

        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \"" + commonAMTest.getRspecTypeName(getTestConfig()) + "\" \"" + commonAMTest.getRspecTypeVersion(getTestConfig()) + "\". " +
                    "This test does not support this, and will not check the validity of the result.");
            return rspec;
        }

        int nodeCount = CommonAMTest.testValidGeni3AdvertisementRspec(this, rspec);
        note("The received RSpec has a length of " + rspec.length() + " characters, and lists " + nodeCount + " nodes.");

        return rspec;
    }

    @Test(hardDepends = {"testCreateSliceNoSliver"}, groups = {"nosliver", "nonodelogin"})
    public void testListResourcesNoSliverSlice() throws JFedException {
        assert sliceNoSliver != null : "sliceNoSliver is null";

        //check if SliverStatus reports not found as it should
        //check without available option
        AggregateManager2.AggregateManagerReply<String> reply =
                am2.listResources(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceNoSliver), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), null/*available*/, true/*compressed*/, sliceNoSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        String specsText = "ListResources returns an RSpec in GENI standard schema. \n" +
                "When a valid geni_slice_urn option is supplied, the returned RSpec will be a Manifest RSpec of the type corresponding to geni_rspec_version, but in manifest format. \n" +
                "If no resources are allocated to the indicated slice by this aggregate, an empty RSpec should be returned -- aggregates should NOT return an error.";

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );

        String rspec = reply.getValue();
        assertNotNull(rspec,
                "ListResources reply is completely empty when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 0,
                "ListResources reply when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified) is not an valid RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );

        assertTrue(CommonAMTest.isEmptyRspec(this, rspec),
                "ListResources reply when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified) is not an empty RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );

        //TODO check if it is a manifest and not a request or advertisement RSpec
    }

    /* same as testListResourcesNoSliverSlice but with available option added. should make no difference. */
    @Test(hardDepends = {"testCreateSliceNoSliver"}, groups = {"nosliver", "nonodelogin"})
    public void testListResourcesNoSliverSliceAvailable() throws JFedException {
        assert sliceNoSliver != null : "sliceNoSliver is null";

        String specsText = "ListResources returns an RSpec in GENI standard schema. \n" +
                "When a valid geni_slice_urn option is supplied, the returned RSpec will be a Manifest RSpec of the type corresponding to geni_rspec_version, but in manifest format. \n" +
                "If no resources are allocated to the indicated slice by this aggregate, an empty RSpec should be returned -- aggregates should NOT return an error.";

        //Test with available TRUE
        AggregateManager2.AggregateManagerReply<String> reply =
                am2.listResources(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceNoSliver), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), true/*available*/, true/*compressed*/, sliceNoSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=true). (according to the specs:\"" + specsText + "\")"
        );

        String rspec = reply.getValue();
        assertNotNull(rspec,
                "ListResources reply is completely empty when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 0,
                "ListResources reply  when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=true) is not an valid RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );

        assertTrue(CommonAMTest.isEmptyRspec(this, rspec),
                "ListResources reply  when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=true) is not an empty RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );


        //Test with available FALSE
        reply =
                am2.listResources(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceNoSliver), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), false/*available*/, true/*compressed*/, sliceNoSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=false). (according to the specs:\"" + specsText + "\")"
        );

        rspec = reply.getValue();
        assertNotNull(rspec,
                "ListResources reply is completely empty when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 0,
                "ListResources reply  when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=false) is not an valid RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );

        assertTrue(CommonAMTest.isEmptyRspec(this, rspec),
                "ListResources reply  when given a slice \"" + sliceNoSliver.urn + "\" that has" +
                        " no sliver (available=false) is not an empty RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );
    }

    @Test(softDepends = {"testGetVersionResultApiVersionsCorrectness",
                         "testGetVersionResultNoDuplicates"}, hardDepends = {
            "testGetVersionXmlRpcCorrectness"}, groups = {"nonodelogin"})
    public void testCreateSliceSliver() throws JFedException, IOException {
        //create a slice to use in AM tests
        String fixedSliceName = getTestConfig().getProperty("fixed_slice_name");
        String fixedProjectName =
                getTestConfig().getProperty("fixed_project_name"); //may be null, in that case, no sub authority is used, unless mandatory
        boolean tryToUseProject =
                TextUtil.objectToBoolean(getTestConfig().getProperty("try_to_use_project_for_slices")) != Boolean.FALSE; //so default is tru

        if (reuseSliceIfExists && fixedSliceName != null) {
            //check if exists: reuseSlice returns info if exists, null otherwise
            sliceSliver = commonAMTest.reuseSlice(fixedSliceName);

            if (sliceSliver != null)
                return;

            //else fall back to creating slice
            if (!createSliceIfNotExists)
                throw new RuntimeException("create_slice_if_not_exists is false, and slice does not exist.");
        }

        if (fixedSliceName == null)
            sliceSliver =
                    commonAMTest.createSlice("s", fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/);
        else
            sliceSliver =
                    commonAMTest.createSlice(fixedSliceName, false, fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/);

        assert sliceSliver != null;
        assert sliceSliver.urn != null : "sliceSliver.urn is null";
        assert sliceSliver.urn.getValue().startsWith("urn:publicid:IDN+");
        assert sliceSliver.credentials != null : "sliceSliver.credential is null";
        assert !sliceSliver.credentials.isEmpty() : "sliceSliver.credential is empty";
    }

    private String requestRspec;

    private Boolean notEnoughResourcesDetected = null;

    @Test(hardDepends = {"testCreateSliceSliver"}, softDepends = {"testListResourcesAvailableNoSlice"}, groups = {
            "createsliver", "createslivernowait", "nonodelogin"})
    public void testCreateSliver() throws JFedException, NoSuchAlgorithmException, IOException {
        assert sliceSliver != null : "sliceSliver is null";

        assert nodeLoginTester == null;
        String fixedSshPublicKeyFile = getTestConfig().getProperty("fixed_ssh_public_key_file");
        String fixedSshPrivateKeyFile = getTestConfig().getProperty("fixed_ssh_private_key_file");
        String fixedSshPrivateKeyPassword = getTestConfig().getProperty("fixed_ssh_private_key_password");
        nodeLoginTester =
                new NodeLoginTester(this, fixedSshPublicKeyFile, fixedSshPrivateKeyFile, fixedSshPrivateKeyPassword);

        String nodeLoginDeadlineMsString = getTestConfig().getProperty("nodelogin_deadline_ms");
        if (nodeLoginDeadlineMsString != null)
            nodeLoginTester.setNodeLoginDeadlineMs(Long.parseLong(nodeLoginDeadlineMsString.trim()));

        String rspecArg = commonAMTest.getFixedRspecOption();
        if (rspecArg == null) {
            note("Searching RSpec to use on authority \"" + testedAuthority.getRspecUrn() + "\" (connecting to " + testedAuthority.getServerToConnect().getDefaultComponentManagerUrn() + ") with " +
                    "type=\"" + testedAuthority.getServerForRspecComponentManager().getServerType() + "\"");
            requestRspec =
                    commonAMTest.getOneNodeRequestRSpec(testedAuthority.getServerForRspecComponentManager(), sliceSliver.urn);
            if (requestRspec == null) skip("testCreateSliver skipped, because no RSpec example known for " +
                    "authority \"" + testedAuthority.getRspecUrn() + "\" type=\"" + testedAuthority.getServerForRspecComponentManager().getServerType() + "\"");
        } else {
            note("Using RSpec supplied in configuration");
            requestRspec = rspecArg;
            if (!commonAMTest.isGeniFixedRspec())
                fatalError("The specified request RSpec is not a geni v3 RSpec. This test does not yet support that case.");
        }

        //TODO: use these to check generated request
        //fixedNodeUrn
        //fixedNodeSliverType


        //CreateSliver needs to generate it's own SSH keypair and use the public key.
        String sshKey = "todo";
        List<String> sshKeys = new ArrayList<>();
        sshKeys.add(nodeLoginTester.getSshKeyHelper().getSshPublicKeyString());
        UserSpec userSpec = new UserSpec(getUser().getUserUrnString(), sshKeys);
        List<UserSpec> userSpecs = new ArrayList<>();
        userSpecs.add(userSpec);
        AggregateManager2.AggregateManagerReply<String> reply =
                am2.createSliver(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver), sliceSliver.urn.getValue(), requestRspec, userSpecs, commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        if (Objects.equals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_REFUSED) && Objects.equals(reply.getOutput(), "Must delete existing slice first")) {
            warn("Received \"Must delete existing slice first\". Probably, the call succeeded but timed out in a previous attempt. This occurs when the server is very busy. We will continue as if the call succeeded.");

            return;
        }

        notEnoughResourcesDetected = AbstractGeniAggregateManager.isNotEnoughFreeResources(reply);
        if (notEnoughResourcesDetected) {
            warn("There were not enough free resources for this test. Will skip further tests. Full message: " + reply.getOutput());
            return;
        }

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "CreateSliver did not succeed");

        createSliverManifestRspec = reply.getValue();

        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \"" + commonAMTest.getRspecTypeName(getTestConfig()) + "\" \"" + commonAMTest.getRspecTypeVersion(getTestConfig()) + "\". " +
                    "This test does not support this, and will skip some checks because of this.");
            return;
        }


        int manifestNodeCount = CommonAMTest.testValidGeni3ManifestRspec(this, createSliverManifestRspec);
        assertNotEquals(manifestNodeCount, 0,
                "CreateSliver returned a manifest RSpec without any nodes. (Expected at least 1 node) " + createSliverManifestRspec);
        assertEquals(manifestNodeCount, 1,
                "CreateSliver returned a manifest RSpec with an incorrect number of nodes. Expected 1 but got " + manifestNodeCount + " " + createSliverManifestRspec);

        assertNotNull(nodeLoginTester);
        assertFalse(nodeLoginTester.hasParsed());
        setErrorsNotFatal();
        assertTrue(CommonAMTest.sameNodesInRequestAndManifest(this, requestRspec, createSliverManifestRspec));
        setErrorsFatal();
        nodeLoginTester.parseSshInfoFromGeni3ManifestRspec(createSliverManifestRspec, null, null);


        assertFalse(CommonAMTest.isEmptyRspec(this, createSliverManifestRspec),
                "CreateSliver returned and empty RSpec: " + createSliverManifestRspec);

        System.out.println("Created sliver for \"" + sliceSliver.urn.getValue() + "\" manifestRspec=" + createSliverManifestRspec);
    }

    public void validSuccesStatus(AggregateManager2.AggregateManagerReply<AggregateManager2.SliverStatus> reply) {
        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "SliverStatus reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

        assertInstanceOf(reply.getRawResult().get("value"), Map.class,
                "SliverStatus value should be a Map. Not: " + reply.getRawResult().get("value"));

        Map rawValue = (Map) reply.getRawResult().get("value");
        String geniUrn = assertMapContainsNonemptyString(rawValue, "geni_urn");
        assertValidUrn(geniUrn, null); //type slice or sliver allowed
        String urntype = GeniUrn.parse(geniUrn).getEncodedResourceType();
        if (!Objects.equals(urntype, "sliver")) {
            if (Objects.equals(urntype, "slice"))
                note("URN type in SliverStatus is \"slice\", but expecting \"sliver\". This is not in compliance with API. However, the API is unlogical here, and a lot of servers return a slice URN here. And in the AMv3 API, it is slice URN.");
            else
                throw new RuntimeException("urntype is " + urntype + " in " + geniUrn + " but expecting \"sliver\" or \"slice\"");
        }
        String geniStatus = assertMapContainsNonemptyString(rawValue, "geni_status");
        assertValidStatus(geniStatus);
        List geniResources = assertMapContainsList(rawValue, "geni_resources");

        for (Object geniResource : geniResources) {
            assertNotNull(geniResource);
            assertInstanceOf(geniResource, Map.class, "one of the geni_resources is not a Map but a " + geniResource.getClass().getName());
            Map ht = (Map) geniResource;
            String resourceUrn = assertMapContainsNonemptyString(ht, "geni_urn");
            assertValidUrn(resourceUrn, null);
            String resourceStatus = assertMapContainsNonemptyString(ht, "geni_status");
            assertValidStatus(resourceStatus);
            assertMapContainsString(ht, "geni_error");
        }
    }

    @Test(hardDepends = {"testCreateSliver"}, groups = {"nonodelogin"})
    public void testSliverStatusExistingSliver() throws JFedException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected)
            skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        //Test SliverStatus with created sliver
        assert sliceSliver != null : "sliceSliver is null";

        AggregateManager2.AggregateManagerReply<AggregateManager2.SliverStatus> reply =
                am2.sliverStatus(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver), sliceSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        validSuccesStatus(reply);
    }

    @Test(hardDepends = {"testCreateSliver"}, groups = {"nonodelogin"})
    public void testListResourcesExistingSliver() throws JFedException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected)
            skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        //Test ListResources on created sliver
        assert sliceSliver != null : "sliceSliver is null";

        AggregateManager2.AggregateManagerReply<String> reply =
                am2.listResources(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), null/*available*/, true/*compressed*/, sliceSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

        String rspec = reply.getValue();
        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 1,
                "ListResources reply  when given a slice \"" + sliceSliver.urn.getValue() + "\" that has" +
                        " a sliver is not an valid RSpec: " + rspec
        );

        assertFalse(CommonAMTest.isEmptyRspec(this, rspec),
                "ListResources reply  when given a slice \"" + sliceSliver.urn.getValue() + "\" that has" +
                        " a sliver is an empty RSpec: " + rspec
        );
    }

    @Test(hardDepends = {"testCreateSliver"}, softDepends = {"testSliverStatusExistingSliver",
                                                             "testListResourcesExistingSliver"}, groups = {
            "createsliver", "nodelogin", "nonodelogin"})
    public void testCreatedSliverBecomesReady() throws JFedException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected)
            skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        //Test if the sliver ever becomes ready. We wait for maximum 20 minutes.
        assert sliceSliver != null : "sliceSliver is null";

        long now = System.currentTimeMillis();
        long deadline = now + (20 * 60 * 1000);

        int failcount = 0;

        while (now < deadline) {
            AggregateManager2.AggregateManagerReply<AggregateManager2.SliverStatus> reply =
                    am2.sliverStatus(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver), sliceSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

            if (!disableComplianceTests)
                validSuccesStatus(reply);

            Map rawValue = (Map) reply.getRawResult().get("value");
            String geniStatus = assertMapContainsNonemptyString(rawValue, "geni_status");
            //API:  Possible values are: configuring, ready, failed, and unknown
            if (geniStatus.equalsIgnoreCase("ready")) {
                note("testCreatedSliverBecomesReady @ " + new Date() + " -> sliver ready: " + rawValue);
                return;
            }

            if (geniStatus.equalsIgnoreCase("failed")) {
                warn("testCreatedSliverBecomesReady @ " + new Date() + " -> sliver FAILED: " + rawValue);
                failcount++;
            }

            if (failcount >= 2) {
                fatalError("testCreatedSliverBecomesReady @ " + new Date() + " -> sliver FAILED: " + rawValue);
            }

            note("testCreatedSliverBecomesReady @ " + new Date() + "-> sliver not ready: " + geniStatus + ".  Trying again in 30 seconds...");
            try {
                Thread.sleep(30000 /*ms*/);
            } catch (InterruptedException e) { /* Ignore*/ }
            now = System.currentTimeMillis();
        }
        throw new RuntimeException("Sliver did not become ready within 20 minutes!");
    }

    @Test(hardDepends = {"testCreateSliver"}, softDepends = {"testCreatedSliverBecomesReady"}, groups = {"nodelogin",
                                                                                                         "nonodelogin"})
    public void checkManifestOnceSliverIsReady() throws JFedException, IOException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected)
            skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        //fetch manifest once sliver is ready, and compare it. Warn if differs.
        //it might have SSH info now that it did not have before!

        AggregateManager2.AggregateManagerReply<String> reply =
                am2.listResources(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()),
                        null/*available*/, true/*compressed*/,
                        sliceSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "ListResources reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

        String rspec = reply.getValue();


        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \"" + commonAMTest.getRspecTypeName(getTestConfig()) + "\" \"" + commonAMTest.getRspecTypeVersion(getTestConfig()) + "\". " +
                    "This test does not support this, and will skip some checks because of this.");
            return;
        }

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 1,
                "ListResources reply  when given a slice \"" + sliceSliver.urn.getValue() + "\" that has" +
                        " a sliver is not an valid RSpec: " + rspec
        );

        assertFalse(CommonAMTest.isEmptyRspec(this, rspec),
                "ListResources reply  when given a slice \"" + sliceSliver.urn.getValue() + "\" that has" +
                        " a sliver is an empty RSpec: " + rspec
        );

        String manifestRspec = reply.getValue();

        if (createSliverManifestRspec == null || manifestRspec == null || !Objects.equals(manifestRspec, createSliverManifestRspec)) {
            note("Manifest RSpec returned by ListResources after Sliver becomes ready differs from manifest RSpec returned by CreateSliver");

            if (createSliverManifestRspec != null)
                note("Manifest RSpec returned by CreateSliver " +
                        (createSliverManifestRspec.contains("ssh-keys") ? "contains"
                                : "DOES NOT contain") + " 'ssh-keys'." +
                        " length=" + createSliverManifestRspec.length() + " chars. ");
            else
                note("Manifest RSpec returned by CreateSliver is null");

            if (manifestRspec != null) {
                note("Manifest RSpec returned by ListResources after Sliver becomes ready " +
                        (manifestRspec.contains("ssh-keys") ? "contains" : "DOES NOT contain") + " 'ssh-keys'." +
                        " length=" + manifestRspec.length() + " chars.");
                setErrorsNotFatal();
                assertTrue(CommonAMTest.sameNodesInRequestAndManifest(this, requestRspec, manifestRspec));
                setErrorsFatal();
                nodeLoginTester.parseSshInfoFromGeni3ManifestRspec(manifestRspec, null, null);

                if (TextUtil.objectToBoolean(getTestConfig().getProperty("skip_final_manifest_check")) != Boolean.TRUE)
                    nodeLoginTester.checkManifestCorrectness(manifestRspec, testedAuthority);
            } else
                errorNonFatal("Manifest RSpec returned by ListResources after Sliver becomes ready is null");

        }
    }


    //run if createSliver was successful, even if sliver never becomes ready.
//    @Test(hardDepends = {"checkManifestOnceSliverIsReady"}, groups = {"nodelogin"} )
    @Test(hardDepends = {"testCreateSliver"}, softDepends = {"testCreatedSliverBecomesReady",
                                                             "checkManifestOnceSliverIsReady"}, groups = {"nodelogin"})
    public void testNodeLogin() throws JFedException, IOException {
        assertNotNull(notEnoughResourcesDetected);
//        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");
        if (notEnoughResourcesDetected) {
            warn("Test skipped because not enough free resources detected while tying to create the sliver(s)");
            return;
        }

        assert nodeLoginTester != null;
        assert nodeLoginTester.hasParsed();

        commonAMTest.setupProxyForNodeLogin(getTestConfig(), nodeLoginTester, sliceSliver.urn);

        nodeLoginTester.testNodeLogin(!beLessStrict);
    }

    @Test(hardDepends = {"testCreatedSliverBecomesReady"}, softDepends = {"testNodeLogin"}, groups = {"nonodelogin"})
    public void testRenewSliver() throws JFedException {
        assert sliceSliver != null : "sliceSliver is null";

        //Test Renew Sliver. Renew so it expires in 5 minutes
        Date expirationTime = new Date(System.currentTimeMillis() + (5 * 60 * 1000));
        AggregateManager2.AggregateManagerReply<Boolean> reply =
                am2.renewSliver(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver),
                        sliceSliver.urn.getValue(), expirationTime, commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "RenewSliver reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

        assertTrue(reply.getValue(),
                "RenewSliver reply's value (which means success) is not true when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

    }

    /* Test must run after tests that use sliver. Will always run, to cleanup any created sliver. */
    @Test(softDepends = {"testNodeLogin", "testRenewSliver", "testCreatedSliverBecomesReady",
                         "checkManifestOnceSliverIsReady"}, groups = {"createsliver", "nodelogin", "nonodelogin"})
    public void testDeleteSliver() throws JFedException {
        if (disableCleanup) skip("No cleanup needed.");
        if (sliceSliver == null) skip("Skipped because other test created nothing to delete");

        assert sliceSliver != null : "sliceSliver is null";

        System.out.println("DeleteSliver for slice " + sliceSliver.urn.getValue());

        //Test Delete Sliver
        AggregateManager2.AggregateManagerReply<Boolean> reply =
                am2.deleteSliver(getAM2Connection(), commonAMTest.getSliceCredentialListForAM(sliceSliver),
                        sliceSliver.urn.getValue(), commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM2CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "DeleteSliver reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");

        assertTrue(reply.getValue(),
                "DeleteSliver reply's value (which means success) is not true when given a slice \"" + sliceSliver.urn.getValue() + "\" that has a sliver.");
    }

    //TODO we could also test if DeleteSliver worked
}
