package be.iminds.ilabt.jfed.highlevel.jobs.states;

import be.iminds.ilabt.jfed.experiment.Experiment;
import be.iminds.ilabt.jfed.experiment.SfaExperimentPart;
import be.iminds.ilabt.jfed.experiment.tasks.ExperimentTaskStatus;
import be.iminds.ilabt.jfed.highlevel.controller.TaskExecutionFinishedCallback;
import be.iminds.ilabt.jfed.highlevel.jobs.Job;
import be.iminds.ilabt.jfed.highlevel.jobs.State;
import be.iminds.ilabt.jfed.highlevel.model.InternalState;
import be.iminds.ilabt.jfed.highlevel.model.Sliver;
import be.iminds.ilabt.jfed.highlevel.tasks.HighLevelTaskFactory;
import be.iminds.ilabt.jfed.highlevel.tasks.ListSliceResourcesTask;
import be.iminds.ilabt.jfed.highlevel.tasks.StatusTask;
import be.iminds.ilabt.jfed.lowlevel.JFedException;
import be.iminds.ilabt.jfed.lowlevel.api_wrapper.StatusDetails;
import be.iminds.ilabt.jfed.rspec.model.RspecNode;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

/**
 * This state consists out of two loops:
 * <p/>
 * - Inner loop does a StatusTask, and repeats it until it reaches a final state
 * - Outer loop will perform an extra ListResources when GetStatus reports ok, because sometimes the status changes to a non-final state when querying ListResources
 * <p/>
 * We repeat both loops until both Status and ListResources report a final state
 * <p/>
 * <p/>
 * User: twalcari
 * Date: 11/27/2015
 * Time: 17:08
 */
public class UpdateUntilReadyOrFailTaskState extends State {
    private static final Logger LOG = LoggerFactory.getLogger(UpdateUntilReadyOrFailTaskState.class);
    private static final int DEFAULT_INITIAL_DELAY = 10_000;
    private static final int MAX_SOFT_FAIL_COUNT = 5;
    private static final int MAX_RESOURCES_FAILED_RETRY_COUNT = 5;
    private static final int MAX_BUSY_COUNT = 30 * 6; //30 minutes

    private final Job<?> job;
    private final SfaExperimentPart experimentPart;
    private final Duration maxWaitUntilReady;

    private final Experiment experiment;
    private final HighLevelTaskFactory hltf;
    private int waitForReadyDelay = DEFAULT_INITIAL_DELAY;

    private Instant timeoutTime;
    private int softFailCount = 0;
    private int resourcesFailedRetryCount = 0;
    private int busyCount = 0;
    private int triesCount = 0;
    private final StatusTask statusTask;
    private boolean waitForFirstCheck;


    protected UpdateUntilReadyOrFailTaskState(Job<?> job, SfaExperimentPart experimentPart,
                                              @Nullable Duration maxWaitUntilReady, HighLevelTaskFactory hltf) {
        super(String.format("Waiting for resources on %s to become ready", experimentPart.getName()));
        this.job = job;
        this.experimentPart = experimentPart;
        this.maxWaitUntilReady = maxWaitUntilReady;
        this.hltf = hltf;
        this.experiment = experimentPart.getExperiment();


        statusTask = hltf.status(experiment.getSlice(), experimentPart.getConnectSfaAuthority());
    }

    protected UpdateUntilReadyOrFailTaskState(Job<?> job, SfaExperimentPart experimentPart, HighLevelTaskFactory hltf) {
        this(job, experimentPart, null, hltf);
    }

    private void registerStart() {
        if (maxWaitUntilReady != null)
            timeoutTime = Instant.now().plus(maxWaitUntilReady);
    }

    private void registerStop() {
        timeoutTime = null;
    }

    @Nonnull
    @Override
    protected ExperimentTaskStatus executeState(Job<?> job) throws InterruptedException, JFedException {

        registerStart();


        if (waitForFirstCheck)
            waitForCheck();

        boolean finalState = false;
        boolean fatalStop = false;
        while (!finalState && !fatalStop && !isTimeout()) {
            //4. Wait for the resources to become ready
            experimentPart.setState(InternalState.WAIT_FOR_READY);

            boolean statusIsFinished = false;

            while (!statusIsFinished && !fatalStop && !isTimeout()) {

                //Check status, and keep user updated on the progress
                CountDownLatch statusTaskExecutionReadyLatch = new CountDownLatch(1);

                this.job.submitTask(statusTask, (TaskExecutionFinishedCallback) (task, taskExecution, state) -> statusTaskExecutionReadyLatch.countDown());
                triesCount++;

                int punctuationCount = -1;

                do {
                    updateMessage(String.format("Checked status %d times. Checking%s",
                            triesCount,
                            Strings.repeat(".", 1 + ++punctuationCount % 3)));
                } while (!statusTaskExecutionReadyLatch.await(1, TimeUnit.SECONDS));


                /* Check all factors that can influence or decision to keep requesting the status.
                If one of these factors is not OK anymore, set fatalStop to TRUE to halt the process. */

                if (statusTask.isBusy()) {
                    busyCount++;

                    if (busyCount > MAX_BUSY_COUNT) {
                        LOG.warn("Got BUSY too many times for resources on {} of {}", experimentPart.getName(), experiment.getName());
                        updateMessage("Waited too long for resources on '" + experimentPart.getName() + "' to become ready." +
                                " (Waited " + busyCount + " times 10 seconds.)");
                        fatalStop = true;
                    }
                }

                if (statusTask.isHardFailure())
                    fatalStop = true;
                else if (statusTask.isSoftFailure()) {
                    softFailCount++;

                    if (softFailCount > MAX_SOFT_FAIL_COUNT) {
                        LOG.warn("Something went wrong while waiting for resources on '\"+experimentPart.getName()+\"' to become ready." +
                                " Will stop checking status...");
                        updateMessage("Fetching status failed. Aborting checks!");

                        fatalStop = true;
                    }
                } else {
                    assert statusTask.getStatusDetails() != null;

                    if (hasAtLeastOneFailure(statusTask.getStatusDetails())) {
                        resourcesFailedRetryCount++;

                        if (resourcesFailedRetryCount > MAX_RESOURCES_FAILED_RETRY_COUNT) {
                            LOG.warn("At least one resource has FAILED  on '{}'. Retried {} times, but there's no point to keep retrying." +
                                    " Will stop checking status...", experimentPart.getName(), MAX_RESOURCES_FAILED_RETRY_COUNT);
                            fatalStop = true;

                            updateMessage("One or more resources have permanently failed");

                        }
                    } else {
                        resourcesFailedRetryCount = 0;
                    }
                }

                statusIsFinished = isStatusTaskFinished();

                if (!statusIsFinished)
                    waitForCheck();
            }

            //log why we stopped listening
            LOG.info("Stopped checking status for {} because {}.",
                    experimentPart.getName(),
                    statusIsFinished ? "statusIsFinalState"
                            : (fatalStop ? "fatalStop" : (isTimeout() ? "timeout" : "UNKNOWN")));

            if (statusIsFinished) {
                LOG.debug("Stopped listening for results of status calls after receiving status {} ",
                        statusTask.getStatusDetails() != null ? statusTask.getStatusDetails().getGlobalStatus()
                                : "null");

                //5. Do a list-resources to verify what we got
                ListSliceResourcesTask sliceResourcesTask =
                        hltf.listSliceResources(experiment.getSlice(), experimentPart.getConnectSfaAuthority());

                CountDownLatch listResourcesTaskReadyLatch = new CountDownLatch(1);
                int punctuationCount = -1;

                this.job.submitTask(sliceResourcesTask, (TaskExecutionFinishedCallback) (task, taskExecution, state) -> listResourcesTaskReadyLatch.countDown());

                do {
                    updateMessage(String.format("Verifying final state%s",
                            Strings.repeat(".", 1 + ++punctuationCount % 3)));
                } while (!listResourcesTaskReadyLatch.await(1, TimeUnit.SECONDS));
                updateMessage("");

                finalState = isAllResourcesReady();

                if (!finalState) {
                    LOG.warn("ListResources reported that not all resources from {} are in a final state. Will start checking again.", experimentPart.getName());
                }
            }
        }
        registerStop();

        return finalState ? ExperimentTaskStatus.SUCCESS : ExperimentTaskStatus.FAILED;
    }

    public boolean isWaitForFirstCheck() {
        return waitForFirstCheck;
    }

    public void setWaitForFirstCheck(boolean waitForFirstCheck) {
        this.waitForFirstCheck = waitForFirstCheck;
    }

    private void waitForCheck() throws InterruptedException {
        //Wait the requested time before querying status
        int waited = 0;
        while (waited < waitForReadyDelay) {
            updateMessage(String.format("Checked status %d times. Next check in %d seconds",
                    triesCount,
                    roundUp((waitForReadyDelay - waited), 1000)));

            Thread.sleep(1000);
            waited += 1000;
        }
    }

    private boolean isStatusTaskFinished() {
        if (statusTask.isBusy())
            return false;

        if (statusTask.getStatusDetails() == null)
            return false;


        //explicitly check all sub-statuses, because the general status is not always correct (FEDIBBTDEV-2178)
        for (StatusDetails.SliverStatus sliverStatus :
                Iterables.concat(statusTask.getStatusDetails().getStatusBySliverUrn().values(),
                        statusTask.getStatusDetails().getStatusByComponentUrn().values())) {
            if (sliverStatus == StatusDetails.SliverStatus.UNKNOWN || sliverStatus == StatusDetails.SliverStatus.CHANGING)
                return false;
        }

        if (statusTask.getStatusDetails().getGlobalStatus() == StatusDetails.SliverStatus.UNKNOWN)
            return false;
        if (statusTask.getStatusDetails().getGlobalStatus() == StatusDetails.SliverStatus.CHANGING)
            return false;

        return true;
    }


    private static boolean hasAtLeastOneFailure(@Nonnull StatusDetails statusDetails) {
        for (StatusDetails.SliverStatus sliverStatus : statusDetails.getStatusBySliverUrn().values())
            if (sliverStatus == StatusDetails.SliverStatus.FAIL)
                return true;
        for (StatusDetails.SliverStatus sliverStatus : statusDetails.getStatusByComponentUrn().values())
            if (sliverStatus == StatusDetails.SliverStatus.FAIL)
                return true;

        return false;
    }


    private boolean isAllResourcesReady() {
        assert experiment.getSlice() != null;
        List<Sliver> sliversOfAuthority = experiment.getSlice().findSlivers(experimentPart.getConnectSfaAuthority());


        for (Sliver sliver : sliversOfAuthority) {
            if (sliver.getNodes() == null) {
                LOG.debug("No nodes available in sliver {}", sliver.getUrn());
                continue;
            }

            final StatusDetails.SliverStatus statusBySliverUrn =
                    experiment.getSlice().getStatus().getStatusBySliverUrn(sliver.getUrn());
            if(statusBySliverUrn == StatusDetails.SliverStatus.CHANGING ||
                    statusBySliverUrn == StatusDetails.SliverStatus.UNKNOWN)
                return false;

            for (RspecNode rspecNode : sliver.getNodes()) {
                if (experiment.getSlice().getStatus().getStatusByComponentUrn().containsKey(rspecNode.getComponentId())) {
                    StatusDetails.SliverStatus nodeStatus =
                            experiment.getSlice().getStatus().getStatusByComponentUrn().get(rspecNode.getComponentId());
                    if (nodeStatus == StatusDetails.SliverStatus.CHANGING
                            || nodeStatus == StatusDetails.SliverStatus.UNKNOWN)
                        return false;
                } else
                    LOG.debug("No status by componentId for {}", rspecNode.getComponentId());
            }
        }
        return true;

    }

    public boolean isTimeout() {
        return timeoutTime != null && timeoutTime.isBefore(Instant.now());
    }

    private static long roundUp(long num, long divisor) {
        return (num + divisor - 1) / divisor;
    }

    public void setWaitForReadyDelay(int waitForReadyDelay) {
        this.waitForReadyDelay = waitForReadyDelay;
    }


}
