package be.iminds.ilabt.jfed.highlevel.jobs;

import be.iminds.ilabt.jfed.TestAuthoritiesModule;
import be.iminds.ilabt.jfed.call_log_output.SerializableApiCallDetails;
import be.iminds.ilabt.jfed.highlevel.jobs.report.JobStateReport;
import be.iminds.ilabt.jfed.highlevel.jobs.states.JobStateFactory;
import be.iminds.ilabt.jfed.highlevel.stitcher.StitchingTestSetupConfig;
import be.iminds.ilabt.jfed.highlevel.stitcher.StitchingTestSetupConfigBuilder;
import be.iminds.ilabt.jfed.lowlevel.GeniAMResponseCode;
import be.iminds.ilabt.jfed.lowlevel.TestCertHelper;
import be.iminds.ilabt.jfed.fedmon.webapi.service.json.*;
import be.iminds.ilabt.jfed.lowlevel.stitching.StitchingDirector;
import be.iminds.ilabt.jfed.lowlevel.stitching.VlanRange;
import javafx.embed.swing.JFXPanel;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;

/**
 * User: twalcari
 * Date: 12/9/2015
 * Time: 10:55
 */

public class StartStitchedExperimentJobIT {

    @BeforeClass
    public void setUp() throws Exception {
        //initialize JavaFX
        new JFXPanel();
    }


    @Test
    public void testScsFailure() throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder();

        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.scsFailure);
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.setAmVersion(1, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setAmVersion(2, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setSupportsAny(1, true);
        confBuilder.setSupportsAny(2, true);
        StitchingTestSetupConfig conf = confBuilder.build();

        //check if builder works correctly
        assertThat(conf.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.scsFailure), equalTo(true));

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(conf);

        setup.testStitching();
        setup.outputToFile();
        setup.assertAllTestBasics();
        assertThat("Did not give up on stitching. globalPhase is " + setup.getAllocateExperimentJob().getState().getName(), setup.isGaveUpOnStitching(), is(true));
        assertThat(setup.getAllocateExperimentJob().getState(), instanceOf(JobStateFactory.StopExperimentState.class));

        assertEquals(setup.getStitchingJobReport().getJobStateReports().stream()
                .filter(jobStateReport -> jobStateReport.getStateName().equals("Request stitched link setup-instructions"))
                .flatMap(jobStateReport -> jobStateReport.getTaskExecutions().stream())
                .flatMap(taskExecution -> taskExecution.getApiCallHistory().stream()).count(), 1, "Expected only the failed SCS call");
    }


    private static List<SerializableApiCallDetails> getSCSCalls(StartStitchedExperimentJobSetup setup) {
        return setup.getStitchingJobReport().getJobStateReports().stream()
                .filter(jobStateReport -> jobStateReport.getStateName().equals("Request stitched link setup-instructions"))
                .flatMap(jobStateReport -> jobStateReport.getTaskExecutions().stream())
                .flatMap(taskExecution -> taskExecution.getApiCallHistory().stream())
                .map(call -> setup.getApiCallDetailsCache().getApiCallDetails(call))
                .collect(Collectors.toList());
    }


    @Test(dataProvider = "allStitchingSetupConfigs", dataProviderClass = StitchingTestDataProviders.class)
    public void parallelStitch(StitchingTestSetupConfig conf) throws Exception {
        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(conf);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(conf.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1)/*allowDelete*/);

        JobStateReport allocateStateReport = setup.getStitchingJobReport().getJobStateReports().stream()
                .filter(jobStateReport -> jobStateReport.getStateName().startsWith("Allocate"))
                .findFirst().orElse(null);
        assertNotNull(allocateStateReport);


        List<SerializableApiCallDetails> allocateStateCalls =
                allocateStateReport.getTaskExecutions().stream()
                        .flatMap(taskExecution -> taskExecution.getApiCallHistory().stream())
                        .map(setup.getApiCallDetailsCache()::getApiCallDetails)
                        .collect(Collectors.toList());

        assertThat(allocateStateCalls, hasSize(greaterThan(1)));
        SerializableApiCallDetails firstAllocate = allocateStateCalls.get(0);
        assertThat(firstAllocate.getGeniMethodName(), isOneOf("CreateSliver", "Allocate"));
        assertThat(firstAllocate.getXmlRpcRequestHumanReadableString(), containsString("<suggestedVLANRange>any</suggestedVLANRange>"));
        if (firstAllocate.getAuthorityUrn().equals(TestCertHelper.am1Urn) && !conf.am(1).supportsAny) {
            if (setup.testAm1Auth.hasFlag(Server.Flag.featureStitchingAny))
                assertThat(firstAllocate.getGeniResponseOutput(), equalTo("ERROR: vlan range any is invalid"));
            else
                assertThat(firstAllocate.getGeniResponseOutput(), not(equalTo("ERROR: vlan range any is invalid")));
        }
        if (firstAllocate.getAuthorityUrn().equals(TestCertHelper.am2Urn) && !conf.am(2).supportsAny) {
            if (setup.testAm2Auth.hasFlag(Server.Flag.featureStitchingAny))
                assertThat(firstAllocate.getGeniResponseOutput(), equalTo("ERROR: vlan range any is invalid"));
            else
                assertThat(firstAllocate.getGeniResponseOutput(), not(equalTo("ERROR: vlan range any is invalid")));
        }
    }

    @Test
    public void testParallelStitcherWithFakeAm3() throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder();

        confBuilder.use3Ams(false);
        confBuilder.setHasFakeAm3(true);

        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.setAmVersion(1, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setAmVersion(2, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setSupportsAny(1, true);
        confBuilder.setSupportsAny(2, true);
        StitchingTestSetupConfig conf = confBuilder.build();

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(conf);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);

        //there should be at least 3 allocate calls. (1 failed and 2 successfull)
        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());

        assertThat(allocateCalls, hasSize(greaterThanOrEqualTo(3)));
    }

    private static List<SerializableApiCallDetails> getAllocateCalls(StartStitchedExperimentJobSetup setup) {
        return setup.getStitchingJobReport().getJobStateReports().stream()
                .filter(jobStateReport -> jobStateReport.getStateName().startsWith("Allocate"))
                .flatMap(jobStateReport -> jobStateReport.getTaskExecutions().stream())
                .flatMap(taskExecution -> taskExecution.getApiCallHistory().stream())
                .map(call -> setup.getApiCallDetailsCache().getApiCallDetails(call))
                .collect(Collectors.toList());
    }

    private static List<SerializableApiCallDetails> getAllCalls(StartStitchedExperimentJobSetup setup) {
        return setup.getStitchingJobReport().getJobStateReports().stream()
                .flatMap(jobStateReport -> jobStateReport.getTaskExecutions().stream())
                .flatMap(taskExecution -> taskExecution.getApiCallHistory().stream())
                .map(call -> setup.getApiCallDetailsCache().getApiCallDetails(call))
                .collect(Collectors.toList());
    }

    @Test
    public void testParallelStitcherWithNonInvolvedAm3() throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder();

        confBuilder.use3Ams(true);
        confBuilder.setNoStitching(3, true);

        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.setAmVersion(1, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setAmVersion(2, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setAmVersion(3, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setSupportsAny(1, true);
        confBuilder.setSupportsAny(2, true);
        StitchingTestSetupConfig conf = confBuilder.build();

        //check if builder works correctly
        assertThat(conf.amCount(), is(3));
        assertThat(conf.am(3).noStitching, is(true));

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(conf);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);

        //there should be at least 3 allocate calls. (1 failed and 2 successfull)
        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);


        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());

        assertThat(allocateCalls, hasSize(greaterThanOrEqualTo(3)));


        Map<String, Integer> am3CommandCounts = getAllCalls(setup).stream()
                .filter(c -> c.getBaseServerUrl().equals(TestAuthoritiesModule.urlC_am3))
                .collect(Collectors.groupingBy(c -> c.getGeniMethodName(), Collectors.summingInt(c -> 1)));
        System.out.println("am3CommandCounts=" + am3CommandCounts);
        assertThat(am3CommandCounts, hasEntry(is("Allocate"), is(1)));
        assertThat(am3CommandCounts, hasEntry(is("Provision"), is(1)));
        assertThat(am3CommandCounts, hasEntry(is("PerformOperationalAction"), is(1)));
        assertThat(am3CommandCounts, hasEntry(is("Status"), is(greaterThanOrEqualTo(1))));
        assertThat(am3CommandCounts, not(hasKey("Delete")));
    }

    @Test(dataProvider = "all32anyCombinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithOneTimeGenericRetryError(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ONETIME_GENERIC_ALLOCATE);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ONETIME_GENERIC_ALLOCATE);

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);

        //there should be at least 3 allocate calls. (1 failed and 2 successfull)
        List<SerializableApiCallDetails> allocateStateCalls =   getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());

        assertThat(allocateCalls, hasSize(greaterThanOrEqualTo(3)));
    }


    @Test(dataProvider = "all32anyCombinations", dataProviderClass = StitchingTestDataProviders.class)
//    @Test(dataProvider = "33anyany-dep") //test with single one
    public void testParallelStitcherWithOneAdvertisementUpToDate(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.setAdvertisementUpToDate(1, false);
        confBuilder.setAdvertisementUpToDate(2, true);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);


        List<SerializableApiCallDetails> allocateStateCalls =  getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> listResourcesCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("ListResources")).collect(Collectors.toList());

        if (!stitchingTestSetupConfig.am(1).supportsAny && stitchingTestSetupConfig.am(1).advUpToDate)
            assertThat(listResourcesCalls, hasSize(2));
        else
            assertThat(listResourcesCalls, hasSize(1));

        //if any not supported, just one call more, as smartness will still be used.
        int callsNeeded = 2;
        if (!stitchingTestSetupConfig.am(1).supportsAny && setup.testAm1Auth.hasFlag(Server.Flag.featureStitchingAny))
            callsNeeded += 1;
        if (!stitchingTestSetupConfig.am(1).supportsAny && !stitchingTestSetupConfig.am(1).advUpToDate)
            callsNeeded += 2;

        assertThat(allocateCalls, hasSize(equalTo(callsNeeded)));

        assertThat(deleteCalls, hasSize(0));
    }


    //TODO  //do an additional ListResources call on am 1 after the call that shows
    //TODO  //    it doesn't support any but does support up to date adv.
    //TODO  // Then it will know which vlan to pick
    //TODO
    //TODO  //This is low prioirty, as this scenario doesn't occur on any AM: All AMs that support up to date adv also support any.
    //TODO
    //TODO After this is implemented, re-enable this test to test it
//    @Test(dataProvider = "all32anyCombinations")
////    @Test(dataProvider = "33anyany-dep") //test with single one
//    public void testParallelStitcherWithBothAdvertisementUpToDate(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
//        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
//        confBuilder.setAm2DependsOnAm1(true);
//        confBuilder.setAdvertisementUpToDate(1, true);
//        confBuilder.setAdvertisementUpToDate(2, true);
//        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
//        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
//
//        StitchingTestSetup setup = new StitchingTestSetup(stitchingTestSetupConfig);
//        setup.testStitching();
//        setup.outputToFile();
//        setup.assertSuccess(true/*allowDelete*/);
//
//        List<SerializableApiCallDetails> allocateStateCalls = setup.getStitchingJobReport().getAllStateCalls("ALLOCATE");
//        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
//                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
//        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
//                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
//        List<SerializableApiCallDetails> listResourcesCalls = allocateStateCalls.stream().filter(
//                c -> c.getGeniMethodName().equals("ListResources")).collect(Collectors.toList());
//
//        if (!stitchingTestSetupConfig.am(1).supportsAny && stitchingTestSetupConfig.am(1).advUpToDate)
//            assertThat(listResourcesCalls, hasSize(2));
//        else
//            assertThat(listResourcesCalls, hasSize(1));
//
//        //if any not supported, just one call more, as smartness will still be used.
//        int callsNeeded = 2;
//        if (!stitchingTestSetupConfig.am(1).supportsAny)
//            callsNeeded += 1;
//        if (!stitchingTestSetupConfig.am(1).supportsAny && !stitchingTestSetupConfig.am(1).advUpToDate)
//            callsNeeded += 2;
//
//        assertThat(allocateCalls, hasSize(equalTo(callsNeeded)));
//
//        assertThat(deleteCalls, hasSize(0));
//    }

    @Test(dataProvider = "all32anyCombinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithOneTimeoutError(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_ONE_TIMEOUT);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_ONE_TIMEOUT);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);

        //there should be at least 3 allocate calls. (1 failed and 2 successfull)
        List<SerializableApiCallDetails> allocateStateCalls =   getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).
                filter(c -> c.getGeniResponseCode() == null || !c.getGeniResponseCode().equals(GeniAMResponseCode.INTERNAL_NONGENI_ERROR.getCode())).
                collect(Collectors.toList());

        assertThat(allocateCalls, hasSize(greaterThanOrEqualTo(3)));
    }

    @Test(dataProvider = "all32anyCombinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithOneAndTwoTimeoutError(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_ONE_TIMEOUT);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_TWO_TIMEOUT);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_ONE_TIMEOUT);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALLOCATE_TWO_TIMEOUT);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);

        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        //there should be at most 2 allocate calls. (1 for AM1 and 1 failed for AM2)
        //unless AM1 does not have any, in which case there are 2 or 3 more calls
        List<SerializableApiCallDetails> allocateStateCalls =getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).
                filter(c -> c.getGeniResponseCode() == null || !c.getGeniResponseCode().equals(GeniAMResponseCode.INTERNAL_NONGENI_ERROR.getCode())).
                collect(Collectors.toList());
        List<SerializableApiCallDetails> nonTimedOutAllocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).
                filter(c -> c.getHttpResponse() != null && !c.getHttpResponse().isEmpty()).
                collect(Collectors.toList());
        List<SerializableApiCallDetails> timedOutAllocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).
                filter(c -> c.getHttpResponse() == null || c.getHttpResponse().isEmpty()).
                collect(Collectors.toList());


        if (stitchingTestSetupConfig.am(1).supportsAny)
            assertThat(nonTimedOutAllocateCalls, hasSize(equalTo(1)));
        else {
            int callsNeeded = 2; //failed AM1 calls
            callsNeeded += 1; //succesfull AM 1 call
//            callsNeeded += 2; //2 timed out AM 2 call

            if (setup.testAm1Auth.hasFlag(Server.Flag.featureStitchingAny))
                callsNeeded += 1; //failed any call to am1

            assertThat(nonTimedOutAllocateCalls, hasSize(equalTo(callsNeeded)));
        }

        assertThat(timedOutAllocateCalls, hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(3))));
    }

    @Test
    public void testParallelStitcherWithAlwaysTimeoutError() throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder();
        confBuilder.setAm2DependsOnAm1(true);
        confBuilder.setAmVersion(1, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setAmVersion(2, StitchingTestSetupConfigBuilder.AmVersion.V3);
        confBuilder.setSupportsAny(1, true);
        confBuilder.setSupportsAny(2, true);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.ERROR_ALWAYS_TIMEOUT);
        StitchingTestSetupConfig conf = confBuilder.build();

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(conf);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        //there should be at most 3 allocate calls. (1 for AM1 and 2 failed for AM2)

        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).
                filter(c -> c.getGeniResponseCode() == null || !c.getGeniResponseCode().equals(GeniAMResponseCode.INTERNAL_NONGENI_ERROR.getCode())).
                collect(Collectors.toList());

        assertThat(allocateCalls, hasSize(lessThanOrEqualTo(3)));
    }

    @Test(dataProvider = "allNoVlanStitchingSetupConfigs", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithNoVlans(StitchingTestSetupConfig stitchingTestSetupConfig) throws Exception {

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        Boolean noVlanAmSupportsAny = null;
        for (int amIndex : new int[]{1, 2}) {
            if (stitchingTestSetupConfig.am(amIndex).hasSpecialReply(StitchingTestSetupConfig.SpecialReply.ERROR_NO_VLANS))
                noVlanAmSupportsAny = stitchingTestSetupConfig.am(amIndex).supportsAny;
        }
        assert noVlanAmSupportsAny != null;


        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());

        if (noVlanAmSupportsAny) {
            //Since this is a permanent, unrecoverable failure, the ALLOCATE phase should give up immediately.
            //If there are more then 4 calls, that is a sure sign that the ALLOCATE call retried up to 3 times.

            //but this assumption is only valid for am1, as am2 will get direct VLAN offers due to its dep on am1

            if (stitchingTestSetupConfig.am(1).hasSpecialReply(StitchingTestSetupConfig.SpecialReply.ERROR_NO_VLANS))
                assertThat(allocateCalls, hasSize(both(greaterThan(0)).and(lessThanOrEqualTo(2))));
        } else {
            //since any is not suppoorted, each tried vlan will be rejected. So we should see many calls.
            assertThat(allocateCalls, hasSize(greaterThan(4)));

        }
    }

    @Test(dataProvider = "all32anyCombinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithMNoMatchVlans(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am1NoMatchingVlansAm2);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am1NoMatchingVlansAm2);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        List<SerializableApiCallDetails> allocateStateCalls =  getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateStateStatusCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().contains("Status")).collect(Collectors.toList());

        int am1vlanCount = new VlanRange(setup.realAvailableAm1).size();
        //the stitcher aslo has a maximum number of VLANs it will try before giving up
        if (am1vlanCount > StitchingDirector.MAX_UNAVAILABLE_VLANS_BEFORE_GIVEUP)
            am1vlanCount = StitchingDirector.MAX_UNAVAILABLE_VLANS_BEFORE_GIVEUP;

        boolean hasFailedAnyCallForAm1 =
                !stitchingTestSetupConfig.am(1).supportsAny && setup.testAm1Auth.hasFlag(Server.Flag.featureStitchingAny);

        assertThat("There should be at least 2 calls for every vlans on AM 1 + 1 if any is not supported",
                allocateCalls, hasSize(greaterThanOrEqualTo(am1vlanCount * 2 + (hasFailedAnyCallForAm1 ? 1 : 0))));

    }

    //TODO: reenable this test if the race condition is fixed
    @Test(dataProvider = "all32Combinations", dataProviderClass = StitchingTestDataProviders.class, enabled = false)
    public void testParallelStitcherWithSingleStatusFailedUnknownAm1(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        //TODO: this test is bad: it doesn't fail for some race conditions: if the sliver that fails was being deleted when it failed (because of AM dep).
        //TODO: make AM 2 fail to prevent this scenario
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.addSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_GENERICCAUSE);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_GENERICCAUSE);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        List<SerializableApiCallDetails> allocateStateCalls =    getAllocateCalls(setup);
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateStateStatusCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().contains("Status")).collect(Collectors.toList());
        long allocateCallsAm1Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am1Urn)).count();
        long allocateCallsAm2Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am2Urn)).count();

        //TODO count all delete calls
//        assertThat(deleteCallsAm1Count, is(1l));
//        assertThat(deleteCallsAm2Count, is(1l));
        //cannot be assumed in all cases:
//        assertThat(allocateCallsAm1Count, is(stitchingTestSetupConfig.am(1).supportsAny ? 1l : 2l));
//        assertThat(allocateCallsAm2Count, is(stitchingTestSetupConfig.am(2).supportsAny ? 1l : 2l));

    }

    @Test(dataProvider = "all32Combinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithSingleStatusFailedVlanUnavailableAm1(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.addSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_VLAN_UNAVAILABLE);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_VLAN_UNAVAILABLE);

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertSuccess(true/*allowDelete*/);


        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateStateStatusCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().contains("Status")).collect(Collectors.toList());
        long deleteCallsAm1Count =
                deleteCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am1Urn)).count();
        long deleteCallsAm2Count =
                deleteCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am2Urn)).count();
        long allocateCallsAm1Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am1Urn)).count();
        long allocateCallsAm2Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am2Urn)).count();

        //the order of trying vlans is not guaranteed, so we can't test this.
//        long vlanRetries = 3; //300,301 and 302 tested before 305 which succeeds
//        assertThat(deleteCallsAm1Count, is(greaterThanOrEqualTo(vlanRetries))); //might be additional one due to status failure
        assertThat(deleteCallsAm2Count, isOneOf(0l, 1l)); //might be deleted to to Status failure, but due to timing might also not
//        assertThat(allocateCallsAm1Count, isOneOf(vlanRetries + 1l, vlanRetries + 2l)); //might be additional one due to status failure
//        assertThat(allocateCallsAm2Count, isOneOf(vlanRetries + 1l, vlanRetries + 2l)); //might be additional one due to status failure

    }

    @Test(dataProvider = "all32Combinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithSingleStatusFailedBandwidthAm2(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_BANDWIDTH);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.FAIL_ONCE_SECOND_STATUS_BANDWIDTH);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();

        List<SerializableApiCallDetails> allocateStateCalls =    getAllocateCalls(setup)    ;
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateStateStatusCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().contains("Status")).collect(Collectors.toList());
        long allocateCallsAm1Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am1Urn)).count();
        long allocateCallsAm2Count =
                allocateCalls.stream().filter(c -> c.getAuthorityUrn().equals(TestCertHelper.am2Urn)).count();

        //TODO count all delete calls
//        assertThat(deleteCallsAm1Count, is(1l));
//        assertThat(deleteCallsAm2Count, is(1l));
        //cannot be assumed in all cases:
//        assertThat(allocateCallsAm1Count, is(stitchingTestSetupConfig.am(1).supportsAny ? 1l : 2l));
//        assertThat(allocateCallsAm2Count, is(stitchingTestSetupConfig.am(2).supportsAny ? 1l : 2l));

    }

    @Test(dataProvider = "all32Combinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithAlwaysStatusFailedAm1(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.addSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ALLWAYS_SECOND_STATUS_VLAN_UNAVAILABLE);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(1, StitchingTestSetupConfig.SpecialReply.FAIL_ALLWAYS_SECOND_STATUS_VLAN_UNAVAILABLE);


        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        setup.assertFailure();
    }

    @Test(dataProvider = "all32Combinations", dataProviderClass = StitchingTestDataProviders.class)
    public void testParallelStitcherWithAlwaysStatusFailedAm2(StitchingTestSetupConfig stitchingTestSetupConfigBase) throws Exception {
        StitchingTestSetupConfigBuilder confBuilder = new StitchingTestSetupConfigBuilder(stitchingTestSetupConfigBase);
        confBuilder.setSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        confBuilder.addSpecialReply(2, StitchingTestSetupConfig.SpecialReply.FAIL_ALLWAYS_SECOND_STATUS_VLAN_UNAVAILABLE);
        StitchingTestSetupConfig stitchingTestSetupConfig = confBuilder.build();
        assert stitchingTestSetupConfig.hasSpecialCase(StitchingTestSetupConfig.SpecialCase.am2DependsOnAm1);
        assert stitchingTestSetupConfig.hasSpecialReply(2, StitchingTestSetupConfig.SpecialReply.FAIL_ALLWAYS_SECOND_STATUS_VLAN_UNAVAILABLE);

        StartStitchedExperimentJobSetup setup = new StartStitchedExperimentJobSetup(stitchingTestSetupConfig);
        setup.testStitching();
        setup.outputToFile();
        System.err.flush();
        System.out.println("Tested setup is: " + stitchingTestSetupConfigBase);
        System.out.println("Final phase: " + setup.getAllocateExperimentJob().getState().getName());
        System.out.println("Gave up: " + setup.isGaveUpOnStitching());
        System.out.flush();
        System.err.flush();
        setup.assertFailure();


        List<SerializableApiCallDetails> allocateStateCalls = getAllocateCalls(setup);
        List<SerializableApiCallDetails> deleteCalls = allocateStateCalls.stream().
                filter(c -> c.getGeniMethodName().startsWith("Delete")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().equals("Allocate") || c.getGeniMethodName().equals("CreateSliver")).collect(Collectors.toList());
        List<SerializableApiCallDetails> allocateStateStatusCalls = allocateStateCalls.stream().filter(
                c -> c.getGeniMethodName().contains("Status")).collect(Collectors.toList());
    }


}
