package be.iminds.ilabt.jfed.lowlevel.api.test;

import be.iminds.ilabt.jfed.log.Logger;
import be.iminds.ilabt.jfed.lowlevel.*;
import be.iminds.ilabt.jfed.lowlevel.api.AbstractGeniAggregateManager;
import be.iminds.ilabt.jfed.lowlevel.api.AggregateManager3;
import be.iminds.ilabt.jfed.lowlevel.api.user_spec.UserSpec;
import be.iminds.ilabt.jfed.lowlevel.connection.JFedConnectionProvider;
import be.iminds.ilabt.jfed.lowlevel.connection.SfaConnection;
import be.iminds.ilabt.jfed.lowlevel.resourceid.ResourceUrn;
import be.iminds.ilabt.jfed.preferences.JFedPreferences;
import be.iminds.ilabt.jfed.testing.base.ApiTest;
import be.iminds.ilabt.jfed.util.RFC3339Util;
import be.iminds.ilabt.jfed.util.TargetAuthority;
import be.iminds.ilabt.jfed.util.TextUtil;

import javax.inject.Inject;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.*;

/**
 * TestAggregateManager3
 */
public class TestAggregateManager3 extends ApiTest {
    NodeLoginTester nodeLoginTester;
    private CommonAMTest commonAMTest;
    private AggregateManager3 am3;

    private boolean beLessStrict = false; //turn some failures into warnings
    private Boolean createSliceIfNotExists = true;
    private Boolean reuseSliceIfExists = false;
    private Boolean disableGetVersionComplianceTests = false;
    private Boolean disableCleanup = false;
    private Boolean geniEndTimeOptionForAllocate = true;
    private Boolean geniEndTimeOptionForProvision = true;
    private Map versionRawResult = null;
    private AggregateManager3.VersionInfo versionInfo = null;
    /* Create the slices needed in the next tests. Uses SA, but not AM */
    private CommonAMTest.SliceInfo slice0;
    private CommonAMTest.SliceInfo sliceS;
    private String sliverSUrnStr;
    private ResourceUrn sliverSUrn;
    private boolean foundNodeLoginInProvison = false;

    @Inject
    public TestAggregateManager3(Logger logger, TargetAuthority testedAuthority, GeniUserProvider geniUserProvider, JFedConnectionProvider connectionProvider, TestbedInfoSource testbedInfoSource, JFedPreferences jFedPreferences) {
        super(logger, testedAuthority, geniUserProvider, connectionProvider, testbedInfoSource, jFedPreferences);
    }

    private static List<String> toStringList(String s) {
        List<String> res = new ArrayList<>();
        res.add(s);
        return res;
    }

    private static List<String> toStringList(String s1, String s2) {
        List<String> res = new ArrayList<>();
        res.add(s1);
        res.add(s2);
        return res;
    }

    private static List<String> toStringList(String s1, String s2, String s3) {
        List<String> res = new ArrayList<>();
        res.add(s1);
        res.add(s2);
        res.add(s3);
        return res;
    }

    public String getTestDescription() {
        return "Many Aggregate Manager (Geni AM API v3) Tests. 2 slices and a sliver will be created during the tests. " +
                "The sliver will be deleted. This will not test ListResources.";
    }

    @Override
    public List<String> getOptConfKeys() {
        List<String> res = new ArrayList<>();
        /* fixed_ssh_private_key_file and fixed_ssh_public_key_file option:
              There are 3 cases:
                  option not specified: use randomly generated key that is different each time
                  option specified, with private key file specified: use public and private key from files (public key can be read from PEM certificate)
                  option specified, with private key file specified, AND this file is the same file for both and is a user login file (key+cert file): use public and private key from file.

         */
        res.add("fixed_ssh_public_key_file"); //needed if fixed_ssh_private_key_file is specified (file containing PEM x509certificate is also allowed)
        res.add("fixed_ssh_private_key_file"); //needed if fixed_ssh_public_key_file is specified
        res.add("fixed_ssh_private_key_password"); //may be null,even if fixed_ssh_private_key_file is not (in case no password is needed)

        //speaksfor and delegation support
        CommonAMTest.addCredentialOptions(res);

        res.add("be_less_strict"); //turns some failures into warnings
        res.add("skip_final_manifest_check"); //don't check the manifest too much

        res.add("geni_end_time_option_for_allocate_call");
        res.add("geni_end_time_option_for_provision_call");

        res.add("nodelogin_deadline_ms");

        res.add("create_slice_if_not_exists");
        res.add("reuse_slice_if_exists");
        res.add("disable_compliance_tests"); //test only for bare functionality, do not do extra checks
        res.add("disable_slice_cleanup"); //no deleteSliver call
        res.add("fixed_slice_name"); //uses this slice name. If not specified, will generate a slice name automatically
        res.add("fixed_project_name"); //uses this as the project/subauthority. If not specified, will choose one if needed, or not use one otherwise.
        res.add("try_to_use_project_for_slices"); //if set and no project name set, this will try to find any project for the user, and use it to create slices.
        CommonAMTest.addOptionsForFixedRspec(res);

        CommonAMTest.addOptionsForNodeLoginSshProxy(res);
        CommonAMTest.addOptionsForConnectionSshProxy(res);

        CommonAMTest.addOptionsForRspecTypeVersion(res);

        return res;
    }

    public SfaConnection getAM3Connection() throws JFedException {
        return (SfaConnection) connectionProvider.getConnectionByAuthority(user, testedAuthority.getServerToConnect(), new ApiInfo.Api(ApiInfo.ApiName.GENI_AM, 3));
    }

    /**
     * Check for correctness of a AM3 XmlRpc result. Should be tested for each reply received.
     */
    public void testAM3CorrectnessXmlRpcResult(Map res) {
        Object code = res.get("code");
        Object value = res.get("value");
        Object output = res.get("output");
        assertNotNull(code, "testAM3CorrectnessXmlRpcResult code == null in " + res);
        assertNotNull(value, "testAM3CorrectnessXmlRpcResult value == null in " + res);
        assertInstanceOf(code, Map.class, "testAM3CorrectnessXmlRpcResult code is not Map in " + res);
        Map codeHt = (Map) code;
        Object genicode = codeHt.get("geni_code");
        assertNotNull(genicode, "testAM3CorrectnessXmlRpcResult code does not contain \"geni_code\" in " + res);
        assertEquals(genicode.getClass(), Integer.class, "testAM3CorrectnessXmlRpcResult code.geni_code is not int in " + res);

        int genicodevalue = (Integer) genicode;
        //output should be present if code is not 0
        if (genicodevalue != 0) {
            assertNotNull(output, "testAM3CorrectnessXmlRpcResult: while geni_code is non success (" + genicodevalue + "), output == null in " + res);
            assertEquals(output.getClass(), String.class, "testAM3CorrectnessXmlRpcResult: while geni_code is non success (" + genicodevalue + "), output is not String (it is " + output.getClass().getName() + " with value \"" + output.toString() + "\") in \"+res+\"");
        } else {
            if (output == null)
                note("testAM3CorrectnessXmlRpcResult: while geni_code is success (" + genicodevalue + "), output is ommited. This is allowed by the API.");
            if (output != null && !(output instanceof String))
                warn("testAM3CorrectnessXmlRpcResult: while geni_code is success (" + genicodevalue + "), output is not String (it is " + output.getClass().getName() + " with value \"" + output.toString() + "\"). This is allowed but not recommended by jFed.");
        }
    }

    public void setUp() {
        assertNotNull(user);
        assertNotNull(user.getUserAuthorityServer());

        commonAMTest = new CommonAMTest(this, user.getUserAuthorityServer());
        am3 = new AggregateManager3(logger, jFedPreferences);

        createSliceIfNotExists = TextUtil.objectToBoolean(getTestConfig().getProperty("create_slice_if_not_exists"));
        reuseSliceIfExists = TextUtil.objectToBoolean(getTestConfig().getProperty("reuse_slice_if_exists"));
        disableGetVersionComplianceTests = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_getversion_compliance_tests"));
        disableCleanup = TextUtil.objectToBoolean(getTestConfig().getProperty("disable_slice_cleanup"));
        if (createSliceIfNotExists == null) createSliceIfNotExists = true;
        if (reuseSliceIfExists == null) reuseSliceIfExists = false;
        if (disableGetVersionComplianceTests == null) disableGetVersionComplianceTests = false;
        if (disableCleanup == null) disableCleanup = false;

        if (getTestConfig().get("be_less_strict") != null) {
            Object v = getTestConfig().get("be_less_strict");
            note("be_less_strict option specified: " + v);
            beLessStrict = TextUtil.objectToBoolean(v + "");
            if (beLessStrict)
                note(" be_less_strict activated: turning some failures into warnings");
            else
                note(" be_less_strict is NOT activated");
        }

        if (getTestConfig().get("geni_end_time_option_for_allocate_call") != null) {
            Object v = getTestConfig().get("geni_end_time_option_for_allocate_call");
            geniEndTimeOptionForAllocate = TextUtil.objectToBoolean(v + "");
        }

        if (getTestConfig().get("geni_end_time_option_for_provision_call") != null) {
            Object v = getTestConfig().get("geni_end_time_option_for_provision_call");
            geniEndTimeOptionForProvision = TextUtil.objectToBoolean(v + "");
        }

        commonAMTest.processCredentialOptions(testbedInfoSource, authorityFinder); //handle speaksFor etc

        commonAMTest.setupProxyForConnectionPool(getTestConfig());
    }

    AggregateManager3.AggregateManagerReply<AggregateManager3.VersionInfo> getVersionReply;
    @Test(groups = {"getversion", "nonodelogin"})
    public void testGetVersionXmlRpcCorrectness() throws JFedException {
        getVersionReply = am3.getVersion(getAM3Connection());

        testAM3CorrectnessXmlRpcResult(getVersionReply.getRawResult());

        //GetVersion is the only method where we have to check for something extra in the raw XMLRPC result
        versionRawResult = getVersionReply.getRawResult();
        assertNotNull(versionRawResult);

        if (!disableGetVersionComplianceTests) {
            Object ver =
                    versionRawResult.get("geni_api"); //this is here for backward compatibility with v1. GetVersion returns this on all versions.
            assertNotNull(ver);
            assertTrue(ver instanceof Integer);
            int v = (Integer) ver;
            assertEquals(v, 3, "AM (backward compatibility \"geni_api\" field) is not version 3");
        }


        assertEquals(getVersionReply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "GeniResponse code is not SUCCESS (0)");

        versionInfo = getVersionReply.getValue();
        assertNotNull(versionInfo);
        assertEquals(versionInfo.getApi(), 3, "AM is not version 3");
    }

    /*
    *
    * For testing:
    *   - slice0  is a slice that will have no sliver on the AM
    *   - sliceS  is a slice that will have a sliver on the AM
    *
    * */

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, groups = {"getversion", "nonodelogin"})
    public void testGetVersionResultCorrectness() throws JFedException {
        note("This test does not call GetVersion again, it uses the result received by \"testGetVersionXmlRpcCorrectness\"");
        Map value = (Map) versionRawResult.get("value");

        //check if all required fields are present
        assertEquals(value.get("geni_api"), 3, "AM \"geni_api\" field in GetVersion reply is not version the Integer 3");

        if (disableGetVersionComplianceTests) skip("Compliance testing is disabled. Will skip this test.");

        assertNotNull(value.get("geni_api_versions"), "geni_api_versions not in version result");
        assertTrue(value.get("geni_api_versions") instanceof Map, "geni_api_versions is not a Map but a " + value.get("geni_api_versions").getClass().getName());

        String[] rspecVersionListnames = {"geni_request_rspec_versions", "geni_ad_rspec_versions"};
        for (String name : rspecVersionListnames) {
            assertNotNull(value.get(name), name + " not in GetVersion result");

            Object[] v =
                    assertInstanceOf(value.get(name) , Object[].class, name + " is not a List but a " + value.get(name).getClass().getName());
            assertTrue(v.length > 0, name + " array is empty");
            for (Object o : v) {
                assertTrue(o instanceof Map, name + " array should contain only Map not a " + o.getClass().getName());
                Map t = (Map) o;
                assertMapContainsNonemptyString(t, "type");
                assertMapContainsNonemptyString(t, "version");
                assertMapContainsString(t, "schema");
                assertMapContainsString(t, "namespace");
                Object extensionsO = t.get("extensions");
                assertNotNull(extensionsO);
                Object[] extensions =
                        assertInstanceOf(extensionsO, Object[].class, "value for extensions of " + name + " is not a String but a " + extensionsO.getClass().getName());;
                for (Object e : extensions)
                    assertTrue(e instanceof String, "an extension of \"+name+\" is not a string but a " + e.getClass().getName());
            }
        }
        String[] credentialTypeListnames = {"geni_credential_types"};
        for (String name : credentialTypeListnames) {
            assertNotNull(value.get(name), name + " not in GetVersion result");

            Object[] v = assertInstanceOf(value.get(name), Object[].class, name + " is not a List but a " + value.get(name).getClass().getName());
            assertTrue(v.length > 0, name + " array is empty");
            for (Object o : v) {
                assertTrue(o instanceof Map, name + " array should contain only Map not a " + o.getClass().getName());
                Map t = (Map) o;
                String typ = assertMapContainsNonemptyString(t, "geni_type");
                //version 3 actually requires a String! But that string should contain an integer.
                //TODO support actual integer as well, but with warning
//                int ver = assertContainsInteger(t, "geni_version", "WARNING: GetVersion result - \"geni_credential_types\" - \"geni_version\" is a String, "+
//                        "but it should have been an Integer. (will parse String to Integer and continue)");
                String ver = assertMapContainsNonemptyString(t, "geni_version");
            }
        }

        Object urn = value.get("urn");
        if (urn != null) {
            setErrorsNotFatal();
            assertInstanceOf(urn, String.class, "if URN is defined, it should be a String, not a " + urn.getClass().getName());
            if (urn instanceof String) {
                String urnString = (String) urn;
                String testUrnString = testedAuthority.getServerToConnect().getDefaultComponentManagerUrn();
                if (!Objects.equals(urnString, testUrnString))
                    warn("URN in GetVersion (" + urnString + ") does not match the URN used to start this test (" + testUrnString +
                            "). This could either be a problem on the server, a bad URN in GetVersion, or it could be that " +
                            "the jFed has wrong info about this authority. This failure could result in a failure to allocate " +
                            "resources (as the component_manager urn used will not match this server).");

                setErrorsFatal();
                assertValidUrn(urnString, null, "GetVersion specified \"urn\" as \"" + urnString + "\", but that does not parse as a valid URN.");
            }
            setErrorsFatal();
        } else {
            note("GetVersion does not specify \"urn\" (which contains the component manager urn). This is not mandatory, but it is useful to include.");
        }
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultCorrectness"}, groups = {"getversion", "nonodelogin"})
    public void testGetVersionResultApiVersionsCorrectness() throws JFedException, MalformedURLException {
        if (disableGetVersionComplianceTests) skip("Compliance testing is disabled. Will skip this test.");

        note("This test does not call GetVersion again, it uses the result received by \"testGetVersionXmlRpcCorrectness\"");
        Map value = (Map) versionRawResult.get("value");
        Map versions = (Map) value.get("geni_api_versions");

        //test if contains self reference
        Object o = versions.get(/*new Integer(3)*/"3");
        if (o == null) {
            o = versions.get(3);
            if (o != null)
                warn("geni_api_versions for key=\"3\" (Integer value in String) is null. (Note that for key=3 (Integer) it is \"" + o + "). " +
                        "This test will accept Integer instead of String, but the API specifies it should be a String.");
        }
        assertNotNull(o, "geni_api_versions for key=3 is null (tried with both int 3 and string \"3\").");
        assertTrue(o instanceof String, "value for key=\"3\" is not a String but a " + o.getClass().getName());

        boolean serverUrlWithLocalhost = false;
        try {
//            assertNotNull(getVersionReply);
            URL serverUrl = new URL(getVersionReply.getXMLRPCCallDetails().getServerUrl());
            serverUrlWithLocalhost = serverUrl.getHost().equals("localhost") || serverUrl.getHost().equals("127.0.0.1");
        } catch (Exception e) {
            note("Unexpected exception in test. Test will ignore this, but this might be an automated tester bug", e);
        }

        if (!serverUrlWithLocalhost) {
            //test that no url's are incorrect and none have localhost as hostname
            for (Object key : versions.keySet()) {
                assertTrue(key instanceof Integer || key instanceof String, "geni_api_versions keys should be Integer (or String), not " + key.getClass().getName());
                int versionNr;
                if (key instanceof Integer) {
                    warn("geni_api_versions contains a key of type Integer with value " + key + ". " +
                            "This test will accept keys of type Integer instead of String, but the API specifies it should be String.");
                    versionNr = (Integer) key;
                } else {
                    versionNr = Integer.parseInt((String) key);
                }
                Object val = versions.get(key);
                assertTrue(val instanceof String, "geni_api_versions values should be String, not " + val.getClass().getName());
                String urlS = (String) val;
                //check URL
                URL url = new URL(urlS);
                String host = url.getHost();
                if (!beLessStrict) {
                    assertFalse(Objects.equals(host, "localhost"), "Illegal host in URL: " + url + " (host in URL should not be the non-global \"" + host + "\")");
                    assertFalse(Objects.equals(host, "127.0.0.1"), "Illegal host in URL: " + url + " (host in URL should not be the non-global \"" + host + "\")");
                } else {
                    if (host.equals("localhost") || host.equals("127.0.0.1"))
                        warn("Illegal host in URL: " + url + " (host in URL should not be the non-global \"" + host + "\")");
                }
            }
        } else
            note("The URL of the server has localhost. This test will assume that this is a test server, and will therefor NOT warn or fail if the URL's in GetVersion have localhost in them.");
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultApiVersionsCorrectness", "testGetVersionResultCorrectness"}, groups = {"getversion", "nonodelogin"})
    public void testGetVersionResultNoDuplicates() throws JFedException {
        if (disableGetVersionComplianceTests) skip("Compliance testing is disabled. Will skip this test.");

        note("This test does not call GetVersion again, it uses the result received by \"testGetVersionXmlRpcCorrectness\"");
        //Check if RSpecs are unique
        List<AggregateManager3.VersionInfo.RspecVersion> adRspecVersions =
                new ArrayList<>();
        for (AggregateManager3.VersionInfo.RspecVersion rspecVer : versionInfo.getAdRspecVersions()) {
            for (AggregateManager3.VersionInfo.RspecVersion other : adRspecVersions)
                assertFalse(other.equalTypeAndVersion(rspecVer), "VersionInfo Result invalid: Duplicate Rspec type/version pair in supported Advertisement RSpec:" +
                        "type=" + other.getType() + " version=" + other.getVersion() + " VS " +
                        "type=" + rspecVer.getType() + " version=" + rspecVer.getVersion() + " ");
            adRspecVersions.add(rspecVer);
        }

        List<AggregateManager3.VersionInfo.RspecVersion> reqRspecVersions =
                new ArrayList<>();
        for (AggregateManager3.VersionInfo.RspecVersion rspecVer : versionInfo.getRequestRspecVersions()) {
            for (AggregateManager3.VersionInfo.RspecVersion other : reqRspecVersions)
                assertFalse(other.equalTypeAndVersion(rspecVer), "VersionInfo Result invalid: Duplicate Rspec type/version pair in supported Request RSpec" +
                        "type=" + other.getType() + " version=" + other.getVersion() + " VS " +
                        "type=" + rspecVer.getType() + " version=" + rspecVer.getVersion() + " ");
            reqRspecVersions.add(rspecVer);
        }

        List<AggregateManager3.VersionInfo.CredentialType> credentialTypes =
                new ArrayList<>();
        for (AggregateManager3.VersionInfo.CredentialType credentialType : versionInfo.getCredentialTypes()) {
            for (AggregateManager3.VersionInfo.CredentialType other : credentialTypes)
                assertFalse(Objects.equals(other, credentialType), "VersionInfo Result invalid: Duplicate CredentialType type/version pair in supported Request RSpec" +
                        "type=" + other.getType() + " version=" + other.getVersion() + " VS " +
                        "type=" + credentialType.getType() + " version=" + credentialType.getVersion() + " ");
            credentialTypes.add(credentialType);
        }
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {"testGetVersionResultNoDuplicates",
            "testGetVersionResultApiVersionsCorrectness"}, groups= {"nonodelogin"},
            description = "This test calls ListResources without any credentials. That should fail.")
    public void testListResourcesBadCredential() throws JFedException {
        //Test without credentials. Should fail.
        AggregateManager3.AggregateManagerReply<String> reply = am3.listResources(
                getAM3Connection(),
                new ArrayList<>(), //no credential
                commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()),
                true,
                true,
                null);

        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        assertFalse(reply.getGeniResponseCode().isSuccess(), "The ListResources call should fail when given no credentials. Instead, it returned success!");

        //only warning if somewhat acceptable error code
        if (reply.getGeniResponseCode().equals(GeniAMResponseCode.GENIRESPONSE_FORBIDDEN) || reply.getGeniResponseCode().equals(GeniAMResponseCode.GENIRESPONSE_ERROR))
            warn("The ListResources call should fail with error code 1 \"Bad Arguments: malformed arguments\" when given no credentials. Instead, it returned " + reply.getGeniResponseCode() + "  " + reply.getGeniResponseCode().getDescription());
        else
            assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_BADARGS, "The ListResources call should fail with error code 1 \"Bad Arguments: malformed arguments\" when given no credentials. Instead, it returned " + reply.getGeniResponseCode() + "  " + reply.getGeniResponseCode().getDescription());
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {"testGetVersionResultNoDuplicates",
            "testGetVersionResultApiVersionsCorrectness"}, groups= {"nonodelogin"})
    public void testListAvailableResources() throws JFedException {
        AggregateManager3.AggregateManagerReply<String> reply = am3.listResources(
                getAM3Connection(),
                commonAMTest.getUserCredentialListForAM(),
                commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()),
                true,
                true,
                commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        assertTrue(reply.getGeniResponseCode().isSuccess(), "The ListResources call did not return success but " + reply.getGeniResponseCode());
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {"testGetVersionResultNoDuplicates",
            "testGetVersionResultApiVersionsCorrectness"}, groups= {"nonodelogin"},
            description = "This test calls Status with user credentials, and the urn of a non exisiting slice. That should fail.")
    public void testStatusBadSlice() throws JFedException {
        String badSliceUrn =
                "urn:publicid:IDN+" + user.getUserAuthorityServer().getUrnTld() + "+slice+NonExisting";

        List<String> urns = new ArrayList<>();
        urns.add(badSliceUrn);

        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply =
                am3.status(getAM3Connection(), urns, commonAMTest.getUserCredentialListForAM(), false/*besteffort*/, commonAMTest.addCredentialExtraOptions(null)/*extraoptions*/);

        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        assertFalse(reply.getGeniResponseCode().isSuccess(), "SliverStatus reply GeniResponse code is SUCCESS (0) when given a non existing slice \"" + badSliceUrn + "\"");
    }

    @Test(hardDepends = {"testGetVersionXmlRpcCorrectness"}, softDepends = {
            "testGetVersionResultNoDuplicates", "testGetVersionResultApiVersionsCorrectness",
            "testStatusBadSlice", "testStatusBadSlice", "testListResourcesBadCredential",
            "testListAvailableResources"},
            description = "Create the slices used in the next tests", groups= {"nonodelogin"})
    public void createTestSlices() throws JFedException, IOException {
        String fixedProjectName = getTestConfig().getProperty("fixed_project_name"); //may be null, in that case, no sub authority is used, unless mandatory
        boolean tryToUseProject = TextUtil.objectToBoolean(getTestConfig().getProperty("try_to_use_project_for_slices"))  != Boolean.FALSE; //so default is tru
        //create slices to use in AM tests
        slice0 = commonAMTest.createSlice("n", fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/); //not actually used with any rspec

        String fixedSliceName = getTestConfig().getProperty("fixed_slice_name");

        if (reuseSliceIfExists && fixedSliceName != null) {
            //check if exists: reuseSlice returns info if exists, null otherwise
            sliceS = commonAMTest.reuseSlice(fixedSliceName);

            if (sliceS != null)
                return;

            //else fall back to creating slice
            if (!createSliceIfNotExists)
                throw new RuntimeException("create_slice_if_not_exists is false, and slice does not exist.");
        }

        if (fixedSliceName == null)
            sliceS = commonAMTest.createSlice("s", fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/);
        else
            sliceS = commonAMTest.createSlice(fixedSliceName, false, fixedProjectName, tryToUseProject, true /*hasFixedRspecOption*/);

        assertNotNull(slice0.urnString, "slice0.urnString may not be null");
        assertTrue(slice0.urnString.startsWith("urn:publicid:IDN+"));
        assertNotNull(slice0.credentials != null, "slice0.credential may not be null");
        assertNotEmpty(slice0.credentials, "slice0.credential may not be empty");
        assertValidUrn(slice0.urnString, "slice");

        assertNotNull(sliceS.urnString, "sliceS.urnString may not be null");
        assertTrue(sliceS.urnString.startsWith("urn:publicid:IDN+"));
        assertNotNull(sliceS.credentials, "sliceS.credential may not be null");
        assertNotEmpty(sliceS.credentials, "sliceS.credential may not be empty");
        assertValidUrn(sliceS.urnString, "slice");
    }

    @Test(hardDepends = {
            "createTestSlices"}, groups= {"nonodelogin"},
            description = "Call Status on a slice without slivers. It is expected an error such as 12 \"Search Failed (eg for slice)\" is returned.")
    public void testStatusNoSliverSlice() throws JFedException {
        //then check if SliverStatus reports not found as it should
        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply =
                am3.status(getAM3Connection(), toStringList(slice0.urnString), commonAMTest.getSliceCredentialListForAM(slice0), false/*besteffort*/, commonAMTest.addCredentialExtraOptions(null));

        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        /*
        * Calling Status() on an unknown, deleted or expired sliver (by explicit URN) shall result in an error
        * (e.g. SEARCHFAILED, EXPIRED or ERROR)
        *
        * (unless geni_best_effort is true, in which case the method may succeed,
         * but return a geni_error for each sliver that failed).
          *
          * Attempting to get Status() for a slice (no slivers identified) with no current slivers at this aggregate
          * may return an empty list for geni_slivers, may return a list of previous slivers that have since been
          * deleted, or may even return an error (e.g. SEARCHFAILED or EXPIRED).
          * Note therefore that geni_slivers may be an empty list.
        * */

        String spec =
                "Attempting to get Status() for a slice (no slivers identified) with no current slivers at this aggregate\n" +
                        " may return an empty list for geni_slivers, may return a list of previous slivers that have since been\n" +
                        " deleted, or may even return an error (e.g. SEARCHFAILED or EXPIRED).\n" +
                        " Note therefore that geni_slivers may be an empty list.";
        note("AMV3 spec says: \"" + spec + "\"");

        boolean isSuccess = reply.getGeniResponseCode().isSuccess();
        boolean emptySliverList = isSuccess && reply.getValue().getSliverInfo().isEmpty();

        boolean isSearchFailed =
                Objects.equals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SEARCHFAILED);
        boolean isExpired = Objects.equals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_EXPIRED);

        if (isSuccess) {
            assertTrue(emptySliverList,
                    "The reply to the Status call has response code SUCCESS (0) when given a slice \"" + slice0.urnString +
                            "\" that has no sliver. However, the sliver list is not empty, as would be expected " +
                            "(because there are no previous slivers in this slice)"
            );
        } else {
            if (!isSearchFailed && !isExpired)
                warn("The reply to the Status call has response code " + reply.getGeniResponseCode() + " when given a slice \"" +
                        slice0.urnString + "\" that has no sliver. This is acceptable behaviour. However, there are more " +
                        "fitting errors to return, such as code 12 \"Search Failed (eg for slice)\" or code 15 \"Expired\"");
        }
    }

    public void verifyDescribeReplyNoSliver(String urn, AggregateManager3.AggregateManagerReply<AggregateManager3.ManifestInfo> reply) {
        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        String specsText =
                "If a slice urn is supplied and there are no slivers in the given slice at this aggregate, " +
                        "then geni_rspec shall be a valid manifest RSpec, containing zero (0) node or link elements - that is, specifying no resources. " +
                        "geni_slivers may be an empty array, or may be an array of previous slivers that have since been deleted or expired. " +
                        "Calling Describe on one or more sliver URNs that are unknown, deleted or expired shall result in an error (e.g. SEARCHFAILED, EXPIRED or ERROR geni_code). ";

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "Describe reply GeniResponse code is not SUCCESS (0) when given a slice \"" + urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );

        AggregateManager3.ManifestInfo manifestInfo = reply.getValue();
        assertNotNull(manifestInfo,
                "Describe reply is completely empty when given a slice \"" + urn + "\" that has" +
                        " no sliver (available=unspecified). (according to the specs:\"" + specsText + "\")"
        );
        String rspec = manifestInfo.getManifestRspec();


        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \""+commonAMTest.getRspecTypeName(getTestConfig())+"\" \""+commonAMTest.getRspecTypeVersion(getTestConfig())+"\". " +
                    "This test does not support this, and will skip some checks because of this.");
            return;
        }

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, rspec), 0,
                "Describe reply when given a slice \"" + urn + "\" that has" +
                        " no sliver (available=unspecified) is not an valid RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );

        assertTrue(CommonAMTest.isEmptyRspec(this, rspec),
                "Describe reply when given a slice \"" + urn + "\" that has" +
                        " no sliver (available=unspecified) is not an empty RSpec (according to the specs:\"" + specsText + "\"): " + rspec
        );
    }

    @Test(hardDepends = {"createTestSlices"}, groups= {"nonodelogin"})
    public void testDescribeNoSliverSlice() throws JFedException {
        assertNotNull(slice0);

        //check if SliverStatus reports not found as it should
        AggregateManager3.AggregateManagerReply<AggregateManager3.ManifestInfo> reply =
                am3.describe(getAM3Connection(), toStringList(slice0.urnString), commonAMTest.getSliceCredentialListForAM(slice0), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), true/*compressed*/, commonAMTest.addCredentialExtraOptions(null));

        verifyDescribeReplyNoSliver(slice0.urnString, reply);

        //check if SliverStatus reports not found as it should
        reply =
                am3.describe(getAM3Connection(), toStringList(slice0.urnString), commonAMTest.getSliceCredentialListForAM(slice0), commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), false/*compressed*/, commonAMTest.addCredentialExtraOptions(null));

        verifyDescribeReplyNoSliver(slice0.urnString, reply);
    }

    private String requestRspec;
    @Test(hardDepends = {"createTestSlices"}, groups= {"nonodelogin"}, softDepends = {"testStatusNoSliverSlice", "testDescribeNoSliverSlice"})
    public void testAllocate() throws JFedException, NoSuchAlgorithmException, ParseException, IOException {
        assertNotNull(sliceS);

        String rspecArg = commonAMTest.getFixedRspecOption();
        if (rspecArg == null) {
            //TODO: make sure geni 3 is advertised in GetVersion. If not, use another RSpec request format!
            requestRspec = commonAMTest.getOneNodeRequestRSpec(testedAuthority.getServerForRspecComponentManager(), sliceS.urn);
            if (requestRspec == null)
                skip("testCreateSliver skipped, because no RSpec example known for type=\"" + testedAuthority.getServerForRspecComponentManager().getServerType() + "\"");
        } else {
            requestRspec = rspecArg;
            if (!commonAMTest.isGeniFixedRspec())
                fatalError("The specified request RSpec is not a geni v3 RSpec. This test does not yet support that case.");
        }

        String endTime = null;
        if (geniEndTimeOptionForAllocate) {
            Date expireDate = new Date(System.currentTimeMillis() + (20 * 60 * 1000L)); //20 minutes in the future
            endTime = RFC3339Util.dateToRFC3339String(expireDate);
        }

        AggregateManager3.AggregateManagerReply<AggregateManager3.AllocateAndProvisionInfo> reply =
                am3.allocate(getAM3Connection(), sliceS.urnString, commonAMTest.getSliceCredentialListForAM(sliceS), requestRspec, endTime, commonAMTest.addCredentialExtraOptions(null));


        notEnoughResourcesDetected = AbstractGeniAggregateManager.isNotEnoughFreeResources(reply);
        if (notEnoughResourcesDetected) {
            warn("There were not enough free resources for this test. Will skip further tests. Full message: "+reply.getOutput());
            return;
        }

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "Allocate did not succeed");

        AggregateManager3.AllocateAndProvisionInfo allocateAndProvisionInfo = reply.getValue();
        String replyRequestRspec = allocateAndProvisionInfo.getRspec();
        assertNotNull(replyRequestRspec, "Allocate should return a request RSpec" + replyRequestRspec);
        //TODO check if it is a request rspec

        //TODO API question: Allocate returns a request RSpec, why? (also, in the specification, the return struct mentions a manifest rspec)

        List<AggregateManager3.SliverInfo> sliverInfos = allocateAndProvisionInfo.getSliverInfo();
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver allocated");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        sliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");
        sliverSUrn = new ResourceUrn(sliverSUrnStr);
        assertEquals(sliverInfo.getAllocationStatus(), "geni_allocated", "Allocated sliver should be in geni_allocated state");

        try {
            Date expireDate = sliverInfo.getExpiresDate();
            Date now = new Date();
            if (!now.before(expireDate))
                warn("newly allocated sliver does not expire in the future! now=" + now + " expireDate=\"" + expireDate + "\"");
//           assertTrue(now.before(expireDate), "allocated sliver does not expire in the future! now="+now+" expireDate="+expireDate);
        } catch (ParseException e) {
            warn("geni_expire data is NOT a valid RFC3339 date: \"" + sliverInfo.getExpires() + "\"");
        }

        System.out.println("Allocated sliver for \"" + sliceS.urnString + "\" replyRequestRspec=" + replyRequestRspec);
    }

    @Test(hardDepends = {"testAllocate"}, groups= {"nonodelogin"})
    public void testProvision() throws JFedException, NoSuchAlgorithmException, ParseException {
        if (notEnoughResourcesDetected) {
            warn("There were not enough free resources for this test. Skipping...");
            return;
        }
        assertNotNull(sliceS);
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");

        //check GetVersion geni_single_allocation  if it is true, we test with slice URN, instead of sliver URN(s)
        List<String> urns = toStringList(sliverSUrnStr);
        if (versionInfo.isSingleSliceAllocation()) {
            urns = toStringList(sliceS.urnString);
        }

        //Note: currently Utah Emulab's current AM3 doesn't work without this optional option
        //      TODO we should also test without users to replicate this problem!

        assert nodeLoginTester == null;
        String fixedSshPublicKeyFile = getTestConfig().getProperty("fixed_ssh_public_key_file");
        String fixedSshPrivateKeyFile = getTestConfig().getProperty("fixed_ssh_private_key_file");
        String fixedSshPrivateKeyPassword = getTestConfig().getProperty("fixed_ssh_private_key_password");
        nodeLoginTester =
                new NodeLoginTester(this, fixedSshPublicKeyFile, fixedSshPrivateKeyFile, fixedSshPrivateKeyPassword);

        String nodeLoginDeadlineMsString = getTestConfig().getProperty("nodelogin_deadline_ms");
        if (nodeLoginDeadlineMsString != null)
            nodeLoginTester.setNodeLoginDeadlineMs(Long.parseLong(nodeLoginDeadlineMsString.trim()));

        List<String> sshKeys = new ArrayList<>();
        sshKeys.add(nodeLoginTester.getSshKeyHelper().getSshPublicKeyString());
        UserSpec userSpec = new UserSpec(user.getUserUrnString(), sshKeys);
        List<UserSpec> users = new ArrayList<>();
        users.add(userSpec);

        String endTime = null;
        if (geniEndTimeOptionForProvision) {
            Date expireDate = new Date(System.currentTimeMillis() + (60 * 60 * 1000L)); //60 minutes in the future
            endTime = RFC3339Util.dateToRFC3339String(expireDate);
        }

        //TODO: make sure geni 3 is advertised in GetVersion. If not, use another RSpec manifest format!

        AggregateManager3.AggregateManagerReply<AggregateManager3.AllocateAndProvisionInfo> reply =
                am3.provision(getAM3Connection(), urns, commonAMTest.getSliceCredentialListForAM(sliceS),
                        commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), null/*bestEffort*/, endTime/*endTime*/, users/*users*/, commonAMTest.addCredentialExtraOptions(null) /*extraoptions*/);


        if (Objects.equals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_REFUSED) && Objects.equals(reply.getOutput(), "Must delete existing slice first")) {
            warn("Received \"Must delete existing slice first\". Probably, the call succeeded but timed out in a previous attempt. This occurs when the server is very busy. We will continue as if the call succeeded.");

            return;
        }

        notEnoughResourcesDetected = AbstractGeniAggregateManager.isNotEnoughFreeResources(reply);
        if (notEnoughResourcesDetected) {
            warn("There were not enough free resources for this test. Will skip further tests. Full message: " + reply.getOutput());
            return;
        }

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "Provision did not succeed");

        AggregateManager3.AllocateAndProvisionInfo allocateAndProvisionInfo = reply.getValue();
        String manifestRspec = allocateAndProvisionInfo.getRspec();
        assertNotNull(manifestRspec, "Provision should return a manifest RSpec");

        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \"" + commonAMTest.getRspecTypeName(getTestConfig()) + "\" \"" + commonAMTest.getRspecTypeVersion(getTestConfig()) + "\". " +
                    "This test does not support this, and will skip some checks because of this.");
        } else {
            assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, manifestRspec), 1,
                    "Provision did not return a valid RSpec" + manifestRspec);
            assertFalse(CommonAMTest.isEmptyRspec(this, manifestRspec),
                    "Provision returned and empty RSpec: " + manifestRspec);

            setErrorsNotFatal();
            assertTrue(CommonAMTest.sameNodesInRequestAndManifest(this, requestRspec, manifestRspec));
            setErrorsFatal();

            assert !nodeLoginTester.hasParsed();
            foundNodeLoginInProvison = nodeLoginTester.parseSshInfoFromGeni3ManifestRspec(manifestRspec, null, null);
            if (foundNodeLoginInProvison)
                note("Successfully found node login info in Provision reply");
            else
                note("Did not find node login info in Provision reply");
        }
        List<AggregateManager3.SliverInfo> sliverInfos = allocateAndProvisionInfo.getSliverInfo();
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver provisioned");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        String resSliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(resSliverSUrnStr);
        assertValidUrn(resSliverSUrnStr, "sliver");
        assertEquals(resSliverSUrnStr, sliverSUrnStr, "Provisioned result has different URN than request");

//Note: State 3, geni_provisioned, is the state of the sliver allocation after the aggregate begins to instantiate the sliver.
        assertEquals(sliverInfo.getAllocationStatus(), "geni_provisioned", "according to specification, sliver should immediately be in geni_provisioned state after the Provision call");

        try {
            Date expireDate = sliverInfo.getExpiresDate();
            Date now = new Date();
            assertTrue(now.before(expireDate), "provisioned sliver does not expire in the future! now=" + now + " expireDate=" + expireDate);
        } catch (ParseException e) {
            warn("geni_expire data is NOT a valid RFC3339 date: \"" + sliverInfo.getExpires() + "\"");
        }

        //TODO check if manifest rspec is valid

        //TODO check if users are in manifest rspec

        System.out.println("Provisioned sliver for \"" + sliceS.urnString + "\" manifestRspec=" + manifestRspec);
    }

    @Test(hardDepends = {"testProvision"}, groups= {"nonodelogin"})
    public void testSliverBecomesProvisioned() throws JFedException, ParseException {
        if (notEnoughResourcesDetected) { warn("There were not enough free resources for this test. Skipping..."); return; }
        //Test if the sliver ever becomes provisioned. We wait for maximum 20 minutes.
        assertNotNull(sliceS);

        long now = System.currentTimeMillis();
        long deadline = now + (20 * 60 * 1000);
        while (now < deadline) {
            AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply =
                    am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                            commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));

            validSuccesStatus(reply);

            AggregateManager3.StatusInfo statusInfo = reply.getValue();
            List<AggregateManager3.SliverInfo> sliverInfos = statusInfo.getSliverInfo();
            AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);

//Note: State 3, geni_provisioned, is the state of the sliver allocation after the aggregate begins to instantiate the sliver.
            assertNotEquals(sliverInfo.getAllocationStatus(), "geni_unallocated", "sliver became geni_unallocated while waiting for it to be provisioned.");
            assertNotEquals(sliverInfo.getAllocationStatus(), "geni_allocated", "sliver became geni_allocated while waiting for it to be provisioned.");
            assertEquals(sliverInfo.getAllocationStatus(), "geni_provisioned", "sliver is not in geni_provisioned while waiting for it to be provisioned.");

            //geni_failed is a state which will will see a a failure to become ready
            assertNotEquals(sliverInfo.getOperationalStatus(), "geni_failed", "sliver operational state became \"geni_failed\" while waiting for it to be provisioned. We assume this is a failure.");

            String resSliverSUrnStr = sliverInfo.getSliverUrn();
            assertNotNull(resSliverSUrnStr);
            assertValidUrn(resSliverSUrnStr, "sliver");
            assertEquals(resSliverSUrnStr, sliverSUrnStr, "Status sliver URN has different URN than in Allocate and Provision");

            //When waiting for "geni_provisioned" to be completed, the sliver is always in state
            //      geni_pending_allocation
            //   typically, when the sliver is "geni_provisioned" it will become
            //      geni_notready   after which it can be started. (but other states are possible)
            //CONCLUSION: if the sliver is not in "geni_pending_allocation", it is ready
            if (!Objects.equals(sliverInfo.getOperationalStatus(), "geni_pending_allocation")) {
                System.out.println("testSliverBecomesReady -> sliver is now fully provisioned: operational_state=" + sliverInfo.getOperationalStatus());
                return;
            }

            System.out.println("testCreatedSliverBecomesReady -> sliver not fully provisioned: operational_state=" + sliverInfo.getOperationalStatus() + ".  Trying again in 30 seconds...");
            try {
                Thread.sleep(30000 /*ms*/);
            } catch (InterruptedException e) { /* Ignore*/ }
            now = System.currentTimeMillis();
        }
        throw new RuntimeException("Sliver did not become ready within 20 minutes!");
    }

    private Boolean notEnoughResourcesDetected = null;
    @Test(hardDepends = {"testSliverBecomesProvisioned"}, groups= {"nonodelogin"})
    public void testPerformOperationalAction() throws JFedException, NoSuchAlgorithmException, ParseException {
        if (notEnoughResourcesDetected) { warn("There were not enough free resources for this test. Skipping..."); return; }
        assertNotNull(sliceS);
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");

        //check GetVersion geni_single_allocation  if it is true, we may only give a slice URN, no sliver URN
        List<String> urns = toStringList(sliverSUrnStr);
        if (versionInfo.isSingleSliceAllocation()) {
            urns = toStringList(sliceS.urnString);
        }

        String action = "geni_start";

        AggregateManager3.AggregateManagerReply<List<AggregateManager3.SliverInfo>> reply;
        try {
            reply = am3.performOperationalAction(getAM3Connection(), urns, commonAMTest.getSliceCredentialListForAM(sliceS),
                    action,
                    null/*bestEffort*/, commonAMTest.addCredentialExtraOptions(null) /*extraoptions*/);
        } catch (JFedException e) {
            if (beLessStrict) {
                warn("Exception while executing performOperationalAction. Due to be_less_strict, this will be ignored. Message: " + e.getMessage());
                return;
            }
            throw e;
        }

        notEnoughResourcesDetected = AbstractGeniAggregateManager.isNotEnoughFreeResources(reply);
        if (notEnoughResourcesDetected) {
            warn("There were not enough free resources for this test. Will skip further tests. Full message: "+reply.getOutput());
            return;
        }

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS, "PerformOperationalAction did not succeed");

        //special check, this really is an error, but we will make it a warning so dependant tests can continue.
        if (!(reply.getRawResult().get("value") instanceof Object[])) {
            if (beLessStrict)
                note("ERROR: Invalid reply to PerformOperationalAction. The \"value\" of a successful call should be a List (API specifies: \"list of struct\"), but it was: " + reply.getRawResult().get("value").getClass() + ". " +
                        "\nNote: This ERROR has been converted to just a note because of the be_less_strict option");
            else
                warn("ERROR: Invalid reply to PerformOperationalAction. The \"value\" of a successful call should be a List (API specifies: \"list of struct\"), but it was: " + reply.getRawResult().get("value").getClass() + ". " +
                        "\nNote: This ERROR has been converted to a warning so that dependent tests can run.");
            return;
        }
        if (reply.getValue() == null) {
            if (beLessStrict)
                note("ERROR: Invalid reply to PerformOperationalAction." +
                        "\nNote: This ERROR has been converted to a warning so that dependent tests can run.");
            else
                warn("ERROR: Invalid reply to PerformOperationalAction." +
                        "\nNote: This ERROR has been converted to a warning so that dependent tests can run.");
            return;
        }

        List<AggregateManager3.SliverInfo> sliverInfos = reply.getValue();
        assertNotNull(sliverInfos, "jFed failed to process the PerformOperationalAction reply correctly. (this is either an invalid reply to PerformOperationalAction or a jFed bug)");
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver returned for PerformOperationalAction");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        String resSliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(resSliverSUrnStr);
        assertValidUrn(resSliverSUrnStr, "sliver");
        assertEquals(resSliverSUrnStr, sliverSUrnStr, "PerformOperationalAction result has different URN than request");
        sliverSUrn = new ResourceUrn(sliverSUrnStr);
//        assertEquals(sliverInfo.getAllocationStatus(), "geni_allocated", "Provisioned sliver should be in geni_allocated state");

        try {
            Date expireDate = sliverInfo.getExpiresDate();
            Date now = new Date();
            assertTrue(now.before(expireDate), "started sliver does not expire in the future! now=" + now + " expireDate=" + expireDate);
        } catch (ParseException e) {
            warn("geni_expire data is NOT a valid RFC3339 date: \"" + sliverInfo.getExpires() + "\"");
        }

        System.out.println("PerformOperationalAction for \"" + sliceS.urnString + "\" action=" + action);
    }

    public void validSuccesStatus(AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply) throws ParseException {
        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "SliverStatus reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceS.urnString + "\" that has a sliver.");

        assertInstanceOf(reply.getRawResult().get("value"), Map.class,
                "SliverStatus value should be a Map. Not: " + reply.getRawResult().get("value"));

        AggregateManager3.StatusInfo statusInfo = reply.getValue();
        assertNotNull(statusInfo);

        String resSliceUrn = statusInfo.getSliceUrn();
        assertNotNull(resSliceUrn);
        assertValidUrn(resSliceUrn, "slice");
        assertEquals(resSliceUrn, sliceS.urnString, "Status slice urn is not correct slice");

        List<AggregateManager3.SliverInfo> sliverInfos = statusInfo.getSliverInfo();
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver status");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        String resSliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(resSliverSUrnStr);
        assertValidUrn(resSliverSUrnStr, "sliver");
        assertEquals(resSliverSUrnStr, sliverSUrnStr, "Provisioned result has different URN than request");
        sliverSUrn = new ResourceUrn(sliverSUrnStr);

        //TODO check if allocation states are allowed values (note that RSpec can describe additional allowed states)
        assertNotNull(sliverInfo.getAllocationStatus());

        //TODO check if operational states are allowed values (note that RSpec can describe additional allowed states)
        assertNotNull(sliverInfo.getOperationalStatus());

        try {
            Date expireDate = sliverInfo.getExpiresDate();
            Date now = new Date();
            assertTrue(now.before(expireDate), "sliver does not expire in the future! now=" + now + " expireDate=" + expireDate);
        } catch (ParseException e) {
            warn("geni_expire data is NOT a valid RFC3339 date: \"" + sliverInfo.getExpires() + "\"");
        }

        assertNotNull(sliverInfo.getExpires());
    }

    @Test(hardDepends = {"testPerformOperationalAction"}, groups = {"postProvisionTests", "nonodelogin"})
    public void testStatusExistingSliver() throws JFedException, ParseException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");
        assertNotNull(sliceS);
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");


        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply =
                am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                        commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));
        validSuccesStatus(reply);
    }

    public void checkDescribeExistingSliver(AggregateManager3.AggregateManagerReply<AggregateManager3.ManifestInfo> reply, String sliceUrn, String sliverUrn) {
        testAM3CorrectnessXmlRpcResult(reply.getRawResult());

        assertEquals(reply.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "Describe reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceS.urnString + "\" that has a sliver.");

        AggregateManager3.ManifestInfo manifestInfo = reply.getValue();
        assertNotNull(manifestInfo,
                "Describe reply is completely empty when given a provisioned sliver \"" + sliverUrn + "\"");

        assertEquals(manifestInfo.getSliceUrn(), sliceUrn, "Describe reply is for another slice URN than requested");

        String manifestRspec = manifestInfo.getManifestRspec();

        if (!commonAMTest.isRspecTypeGeni3(getTestConfig())) {
            note("The RSpec type configured is NOT \"geni\" \"3\" but \""+commonAMTest.getRspecTypeName(getTestConfig())+"\" \""+commonAMTest.getRspecTypeVersion(getTestConfig())+"\". " +
                    "This test does not support this, and will skip some checks because of this.");
            return;
        }

        assertEquals(CommonAMTest.testValidGeni3ManifestRspec(this, manifestRspec), 1,
                "Describe reply rspec when given a provisioned sliver \"" + sliverUrn + "\" is not an valid RSpec");

        List<AggregateManager3.SliverInfo> sliverInfos = manifestInfo.getSliverInfos();
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver status");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        String resSliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(resSliverSUrnStr);
        assertValidUrn(resSliverSUrnStr, "sliver");
        assertEquals(resSliverSUrnStr, sliverSUrnStr, "Describe reply sliver info has different sliver URN than request");
        sliverSUrn = new ResourceUrn(sliverSUrnStr);
        //TODO check if allocation states are allowed values (note that RSpec can describe additional allowed states)
        assertNotNull(sliverInfo.getAllocationStatus());
        //TODO check if operational states are allowed values (note that RSpec can describe additional allowed states)
        assertNotNull(sliverInfo.getOperationalStatus());
        //TODO check if expires is correct date according to RFC
        assertNotNull(sliverInfo.getExpires());

        setErrorsNotFatal();
        assertTrue(CommonAMTest.sameNodesInRequestAndManifest(this, requestRspec, manifestRspec));
        setErrorsFatal();
        boolean foundNodeLoginInDescribe = nodeLoginTester.parseSshInfoFromGeni3ManifestRspec(manifestRspec, null, null);
        if (foundNodeLoginInDescribe)
            note("Successfully found node login info in Describe reply");
        else
            note("Did not find node login info in Describe reply");
        if (foundNodeLoginInProvison)
            assertTrue(foundNodeLoginInDescribe,
                    "Found node login info in the Provision reply. But this same nod login info was NOT found again in the Describe call done afterwards!");

        if (TextUtil.objectToBoolean(getTestConfig().getProperty("skip_final_manifest_check")) != Boolean.TRUE)
            nodeLoginTester.checkManifestCorrectness(manifestRspec, testedAuthority);
    }

    @Test(hardDepends = {"testPerformOperationalAction"}, groups = {"createslivernowait", "postProvisionTests", "nonodelogin"})
    public void testDescribeProvisionedSliver() throws JFedException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");
        assertNotNull(sliceS);
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");

        AggregateManager3.AggregateManagerReply<AggregateManager3.ManifestInfo> reply =
                am3.describe(getAM3Connection(), toStringList(sliverSUrnStr), commonAMTest.getSliceCredentialListForAM(sliceS),
                        commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), true/*compressed*/, commonAMTest.addCredentialExtraOptions(null));

        assertNotNull(versionInfo);
        if (versionInfo.isSingleSliceAllocation()) {
            String apiText =
                    "    geni_single_allocation: <XML-RPC boolean 1/0, default 0>: When true (not default), and " +
                            "performing one of (Describe, Allocate, Renew, Provision, Delete), such an AM requires you to include " +
                            "either the slice urn or the urn of all the slivers in the same state. If you attempt to run one of " +
                            "those operations on just some slivers in a given state, such an AM will return an error. \n\n" +
                            "   For example, at an AM where geni_single_allocation is true you must Provision all geni_allocated " +
                            "slivers at once. If you supply a list of sliver URNs to Provision that is only 'some' of the " +
                            "geni_allocated slivers for this slice at this AM, then the AM will return an error. Similarly, " +
                            "such an aggregate would return an error from Describe if you request a set of sliver URNs that is " +
                            "only some of the geni_provisioned slivers.";
            note("The server reported geni_single_allocation to be true in the GetVersion reply. This will call Describe" +
                    " with the urn of the single sliver in this slice. According to the API this is valid: \n\n" + apiText);
        }

        //TODO need to test if geni_compressed is correctly handled by server.
//        warn("Unimplemented test: need to test if geni_compressed is correctly handled by server.");

        checkDescribeExistingSliver(reply, sliceS.urnString, sliverSUrnStr);
    }

    @Test(hardDepends = {"testPerformOperationalAction"}, softDepends = {"testStatusExistingSliver",
            "testDescribeProvisionedSliver"}, groups = {
            "createsliver", "nodelogin", "postProvisionTests", "nonodelogin"})
    public void testSliverBecomesStarted() throws JFedException, ParseException {
        //Test if the sliver ever becomes ready. We wait for maximum 20 minutes.
        assertNotNull(sliceS);

        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        long now = System.currentTimeMillis();
        long deadline = now + (20 * 60 * 1000);
        while (now < deadline) {
            AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> reply =
                    am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                            commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));

            validSuccesStatus(reply);

            AggregateManager3.StatusInfo statusInfo = reply.getValue();
            List<AggregateManager3.SliverInfo> sliverInfos = statusInfo.getSliverInfo();
            AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);

            assertNotEquals(sliverInfo.getAllocationStatus(), "geni_unallocated", "sliver became geni_unallocated while waiting for it to start.");
            assertNotEquals(sliverInfo.getAllocationStatus(), "geni_allocated", "sliver became geni_allocated while waiting for it to start.");
            assertEquals(sliverInfo.getAllocationStatus(), "geni_provisioned", "sliver is not in geni_provisioned while waiting for it to start.");

            //geni_failed is a state which will will see a a failure to become ready
            assertNotEquals(sliverInfo.getOperationalStatus(), "geni_failed", "sliver operational state became \"geni_failed\" while waiting for it to start.");

            String resSliverSUrnStr = sliverInfo.getSliverUrn();
            assertNotNull(resSliverSUrnStr);
            assertValidUrn(resSliverSUrnStr, "sliver");
            assertEquals(resSliverSUrnStr, sliverSUrnStr, "Status sliver URN has different URN than in Allocate and Provision");

            //Note: this is not generic: not all AM's will support this state
            if (Objects.equals(sliverInfo.getOperationalStatus(), "geni_ready")) {
                System.out.println("testSliverBecomesReady -> sliver ready: " + sliverInfo.getOperationalStatus());
                return;
            }

            System.out.println("testCreatedSliverBecomesReady -> sliver not ready: " + sliverInfo.getOperationalStatus() + ".  Trying again in 30 seconds...");
            try {
                Thread.sleep(30000 /*ms*/);
            } catch (InterruptedException e) { /* Ignore*/ }
            now = System.currentTimeMillis();
        }
        throw new RuntimeException("Sliver did not become ready within 20 minutes!");
    }


    @Test(hardDepends = {"testPerformOperationalAction"}, softDepends = {"testSliverBecomesStarted"}, groups = {
            "postProvisionTests", "nonodelogin"})
    public void testDescribeReadySliver() throws JFedException {
        assertNotNull(notEnoughResourcesDetected);
        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        assertNotNull(sliceS);
        assertNotNull(sliverSUrnStr);
        assertValidUrn(sliverSUrnStr, "sliver");

        AggregateManager3.AggregateManagerReply<AggregateManager3.ManifestInfo> reply =
                am3.describe(getAM3Connection(), toStringList(sliverSUrnStr), commonAMTest.getSliceCredentialListForAM(sliceS),
                        commonAMTest.getRspecTypeName(getTestConfig()), commonAMTest.getRspecTypeVersion(getTestConfig()), true/*compressed*/, commonAMTest.addCredentialExtraOptions(null));

        checkDescribeExistingSliver(reply, sliceS.urnString, sliverSUrnStr);
    }

    //do test if testPerformOperationalAction is successful, even if sliver never becomes ready
    //@Test(hardDepends = {"testSliverBecomesStarted"}, groups = {"nodelogin"} )
    @Test(hardDepends = {"testPerformOperationalAction"}, softDepends = {"testSliverBecomesStarted",
            "testDescribeReadySliver"}, groups = {
            "nodelogin", "postProvisionTests"})
    public void testNodeLogin() throws JFedException, IOException {
        assertNotNull(notEnoughResourcesDetected);
//        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");
        if (notEnoughResourcesDetected) {
            warn("Test skipped because not enough free resources detected while tying to create the sliver(s)");
            return;
        }

        assert nodeLoginTester != null;

        commonAMTest.setupProxyForNodeLogin(getTestConfig(), nodeLoginTester, sliceS.urn);

        nodeLoginTester.testNodeLogin(!beLessStrict);
    }

    @Test(hardDepends = {"testSliverBecomesStarted"}, softDepends = {"testNodeLogin"}, groups = {"postProvisionTests", "nonodelogin"})
    public void testRenewSliver() throws JFedException, ParseException {
        if (notEnoughResourcesDetected) skip("Test skipped because not enough free resources detected while tying to create the sliver(s)");

        assertNotNull(sliceS);


        //Get current expiration time
        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> replyStatus1 =
                am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                        commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));
        validSuccesStatus(replyStatus1);
        assertEquals(replyStatus1.getValue().getSliceUrn(), sliceS.urnString);
        assertEquals(replyStatus1.getValue().getSliverInfo().size(), 1);
        String startExpirationTimeString = replyStatus1.getValue().getSliverInfo().get(0).getExpires();
        Date startExpirationTime;
        try {
            startExpirationTime = RFC3339Util.iso8601StringToDate(startExpirationTimeString);
        } catch (ParseException e) {
            throw new RuntimeException("geni_expire data is NOT a valid RFC3339 date: \"" + startExpirationTimeString + "\"", e);
        }

        Date sliceExpireTime = AnyCredential.getExpireTime(sliceS.credentials);
        if (sliceExpireTime != null) {
            if (!sliceExpireTime.after(startExpirationTime)) {
                note("Sliver expires at same time as slice. Will renew slice first, so sliver can be then be renewed");
                Date newSliceExpirationTime = new Date(startExpirationTime.getTime() + (10 * 60 * 1000));
                sliceS = commonAMTest.renewSlice(sliceS, newSliceExpirationTime);
            }
        }

        //Test Renew Sliver. Renew so it expires 5 minutes later then before
        Date newSliverExpirationTime = new Date(startExpirationTime.getTime() + (5 * 60 * 1000));
//        Date expirationTime = new Date(System.currentTimeMillis() + (5 * 60 * 1000));
        String newSliverExpirationTimeString =
                RFC3339Util.dateToRFC3339String(newSliverExpirationTime, true, true, true);
        AggregateManager3.AggregateManagerReply<List<AggregateManager3.SliverInfo>> replyRenew = am3.renew(
                getAM3Connection(),
                toStringList(sliceS.urnString),
                commonAMTest.getSliceCredentialListForAM(sliceS),
                newSliverExpirationTimeString,
                null/*bestEffort*/,
                commonAMTest.addCredentialExtraOptions(null));

        testAM3CorrectnessXmlRpcResult(replyRenew.getRawResult());

        assertEquals(replyRenew.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "RenewSliver reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceS.urnString + "\" that has a sliver.");


        assertEquals(replyRenew.getValue().size(), 1);
        String renewResSliverExpirationTimeString = replyRenew.getValue().get(0).getExpires();
        Date renewResSliverExpirationTime = RFC3339Util.rfc3339StringToDate(renewResSliverExpirationTimeString);
        assertEquals(renewResSliverExpirationTime.getTime(), newSliverExpirationTime.getTime(),
                "Sliver expiration time has not been set to the data asked in Renew reply: str: requested=\"" +
                        newSliverExpirationTimeString + "\" set=\"" + renewResSliverExpirationTimeString +
                        "\"  date: requested=\"" + newSliverExpirationTime + "\" set=\"" + renewResSliverExpirationTime + "\""
        );


        //Get status and check if expiration time is indeed set
        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> replyStatus2 =
                am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                        commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));
        validSuccesStatus(replyStatus2);
        assertEquals(replyStatus2.getValue().getSliceUrn(), sliceS.urnString);
        assertEquals(replyStatus2.getValue().getSliverInfo().size(), 1);
        String statusResSliverExpirationTimeString = replyStatus2.getValue().getSliverInfo().get(0).getExpires();
        Date statusResSliverExpirationTime = RFC3339Util.rfc3339StringToDate(statusResSliverExpirationTimeString);
        assertEquals(statusResSliverExpirationTime.getTime(), newSliverExpirationTime.getTime(),
                "Sliver expiration time has not been set to the data asked in Status reply: str: requested=\"" +
                        newSliverExpirationTimeString + "\" set=\"" + statusResSliverExpirationTimeString +
                        "\"  date: requested=\"" + newSliverExpirationTime + "\" set=\"" + statusResSliverExpirationTime + "\""
        );
    }

    /* Test must run after tests that use sliver. Will always run, to cleanup any created sliver. */
    @Test(softDepends = {"testNodeLogin", "testRenewSliver", "testSliverBecomesStarted"},
            groups = {"createsliver", "nodelogin", "postProvisionTests", "nonodelogin"})
    public void testDeleteSliver() throws JFedException {
        if (disableCleanup) skip("No cleanup needed.");
        if (sliceS.urnString == null) skip("Skipped because other test created nothing to delete");

        assertNotNull(sliceS);

        System.out.println("DeleteSliver for slice " + sliceS.urnString);

//        //Test Delete Sliver by sliver URN
//        assert sliverSUrnStr != null : "sliverSUrnStr is null";
//        assertValidUrn(sliverSUrnStr, "sliver");
//        AggregateManager3.AggregateManagerReply<List<AggregateManager3.SliverInfo>> reply1 =
//                am3.delete(getAM3Connection(), toStringList(sliverSUrnStr), commonAMTest.getSliceCredentialList(sliceS), null/*best effort*/, null);

        //Test Delete Sliver by slice URN
        AggregateManager3.AggregateManagerReply<List<AggregateManager3.SliverInfo>> reply2 =
                am3.delete(getAM3Connection(), toStringList(sliceS.urnString), commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));

//        testAM3CorrectnessXmlRpcResult(reply1.getRawResult());
        testAM3CorrectnessXmlRpcResult(reply2.getRawResult());

        assertEquals(reply2.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS,
                "DeleteSliver reply GeniResponse code is not SUCCESS (0) when given a slice \"" + sliceS.urnString + "\" that has a sliver.");

        List<AggregateManager3.SliverInfo> sliverInfos = reply2.getValue();
        assertEquals(sliverInfos.size(), 1, "should be exactly 1 sliver status");
        AggregateManager3.SliverInfo sliverInfo = sliverInfos.get(0);
        String resSliverSUrnStr = sliverInfo.getSliverUrn();
        assertNotNull(resSliverSUrnStr);
        assertValidUrn(resSliverSUrnStr, "sliver");
        assertEquals(resSliverSUrnStr, sliverSUrnStr, "Delete reply sliver info has different sliver URN than request");
        sliverSUrn = new ResourceUrn(sliverSUrnStr);

        // Note that this method should return a struct for each deleted sliver, with the URN of the deleted sliver,
        // the allocation state geni_unallocated, and the time when the sliver was previously set to expire.
        // This method may also return an empty list, if no slivers are at this aggregate in the specified slice.

        //TODO check if allocation state is as expected after delete
        assertNotNull(sliverInfo.getAllocationStatus());
        assertEquals(sliverInfo.getAllocationStatus(), "geni_unallocated");

        //might not be present, so no check:
//        //TODO check if operational state is as expected after delete
//        assertNotNull(sliverInfo.getOperationalStatus());


        //test if DeleteSliver worked

        // Calling Status() on an unknown, deleted or expired sliver (by explicit URN) shall result in an error (e.g.
        // SEARCHFAILED, EXPIRED or ERROR) (unless geni_best_effort is true, in which case the method may succeed, but
        // return a geni_error for each sliver that failed).

        //TODO assert that calling status for the status sliver URN fails


        // Attempting to get Status() for a slice (no slivers identified) with no current slivers at this aggregate may
        // return an empty list for geni_slivers, may return a list of previous slivers that have since been deleted,
        // or may even return an error (e.g. SEARCHFAILED or EXPIRED). Note therefore that geni_slivers may be an empty
        // list.
        AggregateManager3.AggregateManagerReply<AggregateManager3.StatusInfo> replyStatus =
                am3.status(getAM3Connection(), toStringList(sliceS.urnString),
                        commonAMTest.getSliceCredentialListForAM(sliceS), null/*best effort*/, commonAMTest.addCredentialExtraOptions(null));
        testAM3CorrectnessXmlRpcResult(replyStatus.getRawResult());

        if (replyStatus.getGeniResponseCode().isSuccess()) {
            //list empty or with deleted slivers (geni_unallocated)

            assertEquals(replyStatus.getGeniResponseCode(), GeniAMResponseCode.GENIRESPONSE_SUCCESS);
            assertInstanceOf(replyStatus.getRawResult().get("value"), Map.class,
                    "SliverStatus value should be a Map. Not: " + replyStatus.getRawResult().get("value"));
            AggregateManager3.StatusInfo statusInfo = replyStatus.getValue();
            assertNotNull(statusInfo);

            String resSliceUrn = statusInfo.getSliceUrn();
            assertNotNull(resSliceUrn);
            assertValidUrn(resSliceUrn, "slice");
            assertEquals(resSliceUrn, sliceS.urnString, "Status slice urn is not correct slice");

            List<AggregateManager3.SliverInfo> sliverInfosStatus = statusInfo.getSliverInfo();

            if (sliverInfosStatus.isEmpty()) {
                //Ok, it has been deleted
                return;
            }

            //all slivers in this slice should be unallocated
            for (AggregateManager3.SliverInfo si : sliverInfosStatus) {
                String siSliverUrnStr = si.getSliverUrn();
                assertNotNull(siSliverUrnStr);
                assertValidUrn(siSliverUrnStr, "sliver");
                assertEquals(si.getAllocationStatus(), "geni_unallocated");
            }
            //Ok, it has been deleted
            return;
        } else {
            //not success
            //Ok, it has been deleted
            return;
        }

    }

}
