// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;

import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.History;

import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;

/**
 * This moves expired failed nodes:
 * <ul>
 *     <li>To parked: If the node has known hardware failure, docker hosts are moved to parked only when all its
 *     children are already in parked
 *     <li>To dirty: If the node has failed less than 5 times OR the environment is dev, test or perf OR system is CD,
 *     as those environments have no protection against users running bogus applications, so
 *     we cannot use the node failure count to conclude the node has a failure.
 *     <li>Otherwise the node will remain in failed
 * </ul>
 * Failed nodes are typically given a long expiry time to enable us to manually moved them back to
 * active to recover data in cases where the node was failed accidentally.
 * <p>
 * The purpose of the automatic recycling to dirty + fail count is that nodes which were moved
 * to failed due to some undetected hardware failure will end up being failed again.
 * When that has happened enough they will not be recycled.
 * <p>
 * The Chef recipe running locally on the node may set the hardwareFailureDescription to avoid the node
 * being automatically recycled in cases where an error has been positively detected.
 *
 * @author bratseth
 */
public class FailedExpirer extends Expirer {

    private final NodeRepository nodeRepository;
    private final Zone zone;

    public FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, 
                         Duration failTimeout, JobControl jobControl) {
        super(Node.State.failed, History.Event.Type.failed, nodeRepository, clock, failTimeout, jobControl);
        this.nodeRepository = nodeRepository;
        this.zone = zone;
    }

    @Override
    protected void expire(List<Node> expired) {
        List<Node> nodesToRecycle = new ArrayList<>();
        for (Node recycleCandidate : expired) {
            if (recycleCandidate.status().hardwareFailureDescription().isPresent()) {
                boolean shouldBeParked = recycleCandidate.type() != NodeType.host ||
                        nodeRepository.getChildNodes(recycleCandidate.hostname()).stream()
                                .allMatch(node -> node.state() == Node.State.parked);

                if (shouldBeParked) nodeRepository.park(
                        recycleCandidate.hostname(), Agent.system, "Parked by FailedExpirer due to HW failure on node");
            } else if (! failCountIndicatesHwFail(zone, recycleCandidate) || recycleCandidate.status().failCount() < 5) {
                nodesToRecycle.add(recycleCandidate);
            }
        }
        nodeRepository.setDirty(nodesToRecycle);
    }

    private boolean failCountIndicatesHwFail(Zone zone, Node node) {
        if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) return false;
        if (zone.system() == SystemName.cd) return false;
        return zone.environment() == Environment.prod || zone.environment() == Environment.staging;
    }

}

@@ -691,20 +691,119 @@
 va.util.
-List
+Collections;%0Aimport java.util.List;%0Aimport java.util.logging.Logger;%0Aimport java.util.stream.Collectors
 ;%0A%0A/**%0A 
@@ -2040,16 +2040,101 @@
 irer %7B%0A%0A
+    private static final Logger log = Logger.getLogger(NodeRetirer.class.getName());%0A
     priv
@@ -2816,30 +2816,38 @@
     
-boolean shouldBeParked
+List%3CString%3E nonParkedChildren
  = r
@@ -2886,18 +2886,43 @@
 pe.host 
-%7C%7C
+? Collections.emptyList() :
 %0A       
@@ -3042,16 +3042,14 @@
    .
-allMatch
+filter
 (nod
@@ -3066,17 +3066,17 @@
 state() 
-=
+!
 = Node.S
@@ -3087,19 +3087,23 @@
 .parked)
-;%0A%0A
+%0A      
         
@@ -3114,169 +3114,584 @@
     
-if (shouldBeParked) nodeRepository.park(%0A                        recycleCandidate.hostname(), Agent.system, %22Parked by FailedExpirer due to HW failure on node%22);
+          .map(Node::hostname)%0A                                .collect(Collectors.toList());%0A%0A                if (nonParkedChildren.isEmpty()) %7B%0A                    nodeRepository.park(recycleCandidate.hostname(), Agent.system, %22Parked by FailedExpirer due to HW failure on node%22);%0A                %7D else %7B%0A                    log.info(String.format(%22Expired failed node %25s with HW fail is not parked because some of its children%22 +%0A                            %22 (%25s) are not yet parked%22, recycleCandidate.hostname(), String.join(%22, %22, nonParkedChildren)));%0A                %7D
 %0A   

Add log message