feat(ecs): Refactor TaskHealthCachingAgent to use TargetHealthCache
* feat(ecs): Refactor TaskHealthCachingAgent to use TargetHealthCache
* Determines Task health with cached ALB target healths vs.
calls to Elastic Load Balancing
* Return status_unknown for targets not found
MODIFY
clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCachingAgent.java
clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCachingAgent.java
@@ -31,10 +31,6 @@ import com.amazonaws.services.ecs.model.NetworkBinding;
import com.amazonaws.services.ecs.model.NetworkInterface;
import com.amazonaws.services.ecs.model.PortMapping;
import com.amazonaws.services.ecs.model.TaskDefinition;
-import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing;
-import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthRequest;
-import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthResult;
-import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription;
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthDescription;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.netflix.spinnaker.cats.agent.AgentDataType;
@@ -45,14 +41,8 @@ import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
import com.netflix.spinnaker.clouddriver.core.provider.agent.HealthProvidingCachingAgent;
import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
-import com.netflix.spinnaker.clouddriver.ecs.cache.client.ContainerInstanceCacheClient;
-import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient;
-import com.netflix.spinnaker.clouddriver.ecs.cache.client.TaskCacheClient;
-import com.netflix.spinnaker.clouddriver.ecs.cache.client.TaskDefinitionCacheClient;
-import com.netflix.spinnaker.clouddriver.ecs.cache.model.ContainerInstance;
-import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
-import com.netflix.spinnaker.clouddriver.ecs.cache.model.Task;
-import com.netflix.spinnaker.clouddriver.ecs.cache.model.TaskHealth;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.*;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.*;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -107,8 +97,8 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
new TaskDefinitionCacheClient(providerCache, objectMapper);
ServiceCacheClient serviceCacheClient = new ServiceCacheClient(providerCache, objectMapper);
- AmazonElasticLoadBalancing amazonloadBalancing =
- amazonClientProvider.getAmazonElasticLoadBalancingV2(account, region, false);
+ TargetHealthCacheClient targetHealthCacheClient =
+ new TargetHealthCacheClient(providerCache, objectMapper);
ContainerInstanceCacheClient containerInstanceCacheClient =
new ContainerInstanceCacheClient(providerCache);
@@ -148,7 +138,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
if (task.getContainers().get(0).getNetworkBindings().size() >= 1) {
taskHealth =
inferHealthNetworkBindedContainer(
- amazonloadBalancing,
+ targetHealthCacheClient,
task,
containerInstance,
serviceName,
@@ -157,7 +147,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
} else {
taskHealth =
inferHealthNetworkInterfacedContainer(
- amazonloadBalancing, task, serviceName, service, taskDefinition);
+ targetHealthCacheClient, task, serviceName, service, taskDefinition);
}
log.debug("Task Health contains the following elements: {}", taskHealth);
@@ -172,7 +162,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
}
private TaskHealth inferHealthNetworkInterfacedContainer(
- AmazonElasticLoadBalancing amazonloadBalancing,
+ TargetHealthCacheClient targetHealthCacheClient,
Task task,
String serviceName,
Service loadBalancerService,
@@ -205,7 +195,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
overallTaskHealth =
describeTargetHealth(
- amazonloadBalancing,
+ targetHealthCacheClient,
task,
loadBalancerService,
serviceName,
@@ -231,12 +221,14 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
private TaskHealth makeTaskHealth(
Task task, String serviceName, TargetHealthDescription healthDescription) {
+ String targetHealth = STATUS_UNKNOWN;
+ if (healthDescription != null) {
log.debug("Task target health is: {}", healthDescription.getTargetHealth());
- String targetHealth =
+ targetHealth =
healthDescription.getTargetHealth().getState().equals("healthy")
? STATUS_UP
: STATUS_UNKNOWN;
-
+ }
TaskHealth taskHealth = new TaskHealth();
taskHealth.setType("loadBalancer");
taskHealth.setState(targetHealth);
@@ -249,7 +241,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
}
private TaskHealth inferHealthNetworkBindedContainer(
- AmazonElasticLoadBalancing amazonloadBalancing,
+ TargetHealthCacheClient targetHealthCacheClient,
Task task,
ContainerInstance containerInstance,
String serviceName,
@@ -286,7 +278,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
overallTaskHealth =
describeTargetHealth(
- amazonloadBalancing,
+ targetHealthCacheClient,
task,
loadBalancerService,
serviceName,
@@ -299,8 +291,20 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
return overallTaskHealth;
}
+ private TargetHealthDescription findHealthDescription(
+ List<TargetHealthDescription> targetHealths, String targetId, Integer targetPort) {
+
+ return targetHealths.stream()
+ .filter(
+ h ->
+ h.getTarget().getId().equals(targetId)
+ && h.getTarget().getPort().equals(targetPort))
+ .findFirst()
+ .orElse(null);
+ }
+
private TaskHealth describeTargetHealth(
- AmazonElasticLoadBalancing amazonloadBalancing,
+ TargetHealthCacheClient targetHealthCacheClient,
Task task,
Service loadBalancerService,
String serviceName,
@@ -308,25 +312,31 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
String targetId,
Integer targetPort,
TaskHealth overallTaskHealth) {
- DescribeTargetHealthResult describeTargetHealthResult =
- amazonloadBalancing.describeTargetHealth(
- new DescribeTargetHealthRequest()
- .withTargetGroupArn(targetGroupArn)
- .withTargets(new TargetDescription().withId(targetId).withPort(targetPort)));
-
- if (describeTargetHealthResult.getTargetHealthDescriptions().isEmpty()) {
- log.debug("Target health description is empty");
+
+ String targetHealthKey = Keys.getTargetHealthKey(accountName, region, targetGroupArn);
+ EcsTargetHealth targetHealth = targetHealthCacheClient.get(targetHealthKey);
+
+ if (targetHealth == null) {
+ log.debug("Cached EcsTargetHealth is empty for targetGroup {}", targetGroupArn);
evictStaleData(task, loadBalancerService);
- return overallTaskHealth;
+ return makeTaskHealth(task, serviceName, null);
}
+ TargetHealthDescription targetHealthDescription =
+ findHealthDescription(targetHealth.getTargetHealthDescriptions(), targetId, targetPort);
+ if (targetHealthDescription == null) {
log.debug(
- "Target health description is not empty and has a size of {}",
- describeTargetHealthResult.getTargetHealthDescriptions().size());
- TargetHealthDescription healthDescription =
- describeTargetHealthResult.getTargetHealthDescriptions().get(0);
+ "TargetHealthDescription is empty on targetGroup '{}' for {}:{}",
+ targetGroupArn,
+ targetId,
+ targetPort);
+ evictStaleData(task, loadBalancerService);
+ return makeTaskHealth(task, serviceName, null);
+ }
+
+ log.debug("Retrieved health of targetId {} for targetGroup {}", targetId, targetGroupArn);
- TaskHealth taskHealth = makeTaskHealth(task, serviceName, healthDescription);
+ TaskHealth taskHealth = makeTaskHealth(task, serviceName, targetHealthDescription);
if ((overallTaskHealth == null) || (taskHealth.getState().equals(STATUS_UNKNOWN))) {
return taskHealth;
}

MODIFY
clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCacheSpec.groovy
clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCacheSpec.groovy
@@ -25,6 +25,7 @@ import com.amazonaws.services.ecs.model.NetworkBinding
import com.amazonaws.services.ecs.model.PortMapping
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing
import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthResult
+import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealth
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthDescription
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthStateEnum
@@ -38,6 +39,7 @@ import spock.lang.Specification
import spock.lang.Subject
import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TARGET_HEALTHS
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS
@@ -64,10 +66,13 @@ class TaskHealthCacheSpec extends Specification {
def healthKey = Keys.getTaskHealthKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, CommonCachingAgent.TASK_ID_1)
def serviceKey = Keys.getServiceKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, CommonCachingAgent.SERVICE_NAME_1)
def containerInstanceKey = Keys.getContainerInstanceKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, CommonCachingAgent.CONTAINER_INSTANCE_ARN_1)
+ def targetHealthKey = Keys.getTargetHealthKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, targetGroupArn)
ObjectMapper mapper = new ObjectMapper()
Map<String, Object> containerMap = mapper.convertValue(new Container().withNetworkBindings(new NetworkBinding().withContainerPort(1338).withHostPort(1338)), Map.class)
Map<String, Object> loadbalancerMap = mapper.convertValue(new LoadBalancer().withTargetGroupArn(targetGroupArn).withContainerPort(1338), Map.class)
+ Map<String, Object> targetHealthMap = mapper.convertValue(
+ new TargetHealthDescription().withTarget(new TargetDescription().withId(CommonCachingAgent.EC2_INSTANCE_ID_1).withPort(1338)).withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy)), Map.class)
def taskAttributes = [
taskId : CommonCachingAgent.TASK_ID_1,
@@ -98,6 +103,14 @@ class TaskHealthCacheSpec extends Specification {
def containerInstanceCache = new DefaultCacheData(containerInstanceKey, containerInstanceAttributes, Collections.emptyMap())
providerCache.get(Keys.Namespace.CONTAINER_INSTANCES.toString(), containerInstanceKey) >> containerInstanceCache
+ def targetHealthAttributes = [
+ targetGroupArn : targetGroupArn,
+ targetHealthDescriptions : Collections.singletonList(targetHealthMap)
+ ]
+
+ def targetHealthCache = new DefaultCacheData(targetHealthKey, targetHealthAttributes, Collections.emptyMap())
+ providerCache.get(TARGET_HEALTHS.toString(), targetHealthKey) >> targetHealthCache
+
DescribeTargetHealthResult describeTargetHealthResult = new DescribeTargetHealthResult().withTargetHealthDescriptions(
new TargetHealthDescription().withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
)

MODIFY
clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCachingAgentSpec.groovy
clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/provider/agent/TaskHealthCachingAgentSpec.groovy
@@ -28,6 +28,7 @@ import com.amazonaws.services.ecs.model.TaskDefinition
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing
import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthRequest
import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthResult
+import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealth
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthDescription
import com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthStateEnum
@@ -43,6 +44,7 @@ import spock.lang.Specification
import spock.lang.Subject
import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TARGET_HEALTHS
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS
@@ -51,7 +53,6 @@ class TaskHealthCachingAgentSpec extends Specification {
def clientProvider = Mock(AmazonClientProvider)
def providerCache = Mock(ProviderCache)
def credentialsProvider = Mock(AWSCredentialsProvider)
- def amazonloadBalancing = Mock(AmazonElasticLoadBalancing)
def targetGroupArn = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':769716316905:targetgroup/test-target-group/9e8997b7cff00c62'
ObjectMapper mapper = new ObjectMapper()
@@ -60,13 +61,21 @@ class TaskHealthCachingAgentSpec extends Specification {
TaskHealthCachingAgent agent = new TaskHealthCachingAgent(CommonCachingAgent.netflixAmazonCredentials, CommonCachingAgent.REGION, clientProvider, credentialsProvider, mapper)
def setup() {
- clientProvider.getAmazonElasticLoadBalancingV2(_, _, _) >> amazonloadBalancing
def serviceKey = Keys.getServiceKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, CommonCachingAgent.SERVICE_NAME_1)
def containerInstanceKey = Keys.getContainerInstanceKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, CommonCachingAgent.CONTAINER_INSTANCE_ARN_1)
+ def targetHealthKey = Keys.getTargetHealthKey(CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, targetGroupArn)
ObjectMapper mapper = new ObjectMapper()
Map<String, Object> loadbalancerMap = mapper.convertValue(new LoadBalancer().withTargetGroupArn(targetGroupArn).withContainerPort(1338), Map.class)
+ Map<String, Object> targetHealthMap = mapper.convertValue(
+ new TargetHealthDescription().withTarget(new TargetDescription().withId(CommonCachingAgent.EC2_INSTANCE_ID_1).withPort(1338)).withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy)), Map.class)
+ Map<String, Object> targetHealthMap2 = mapper.convertValue(
+ new TargetHealthDescription().withTarget(new TargetDescription().withId("192.168.0.100").withPort(1338)).withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy)), Map.class)
+
+ def targetHealths = new ArrayList<>();
+ targetHealths.add(targetHealthMap)
+ targetHealths.add(targetHealthMap2)
providerCache.filterIdentifiers(_, _) >> []
@@ -86,6 +95,14 @@ class TaskHealthCachingAgentSpec extends Specification {
]
def containerInstanceCache = new DefaultCacheData(containerInstanceKey, containerInstanceAttributes, Collections.emptyMap())
providerCache.get(Keys.Namespace.CONTAINER_INSTANCES.toString(), containerInstanceKey) >> containerInstanceCache
+
+ def targetHealthAttributes = [
+ targetGroupArn : targetGroupArn,
+ targetHealthDescriptions : targetHealths
+ ]
+
+ def targetHealthCache = new DefaultCacheData(targetHealthKey, targetHealthAttributes, Collections.emptyMap())
+ providerCache.get(TARGET_HEALTHS.toString(), targetHealthKey) >> targetHealthCache
}
def 'should get a list of task health'() {
@@ -120,15 +137,6 @@ class TaskHealthCachingAgentSpec extends Specification {
def taskHealthList = agent.getItems(ecs, providerCache)
then:
- amazonloadBalancing.describeTargetHealth({ DescribeTargetHealthRequest request ->
- request.targetGroupArn == targetGroupArn
- request.targets.size() == 1
- request.targets.get(0).id == CommonCachingAgent.EC2_INSTANCE_ID_1
- request.targets.get(0).port == 1338
- }) >> new DescribeTargetHealthResult().withTargetHealthDescriptions(
- new TargetHealthDescription().withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
- )
-
taskHealthList.size() == 1
TaskHealth taskHealth = taskHealthList.get(0)
taskHealth.getState() == 'Up'
@@ -175,15 +183,6 @@ class TaskHealthCachingAgentSpec extends Specification {
def taskHealthList = agent.getItems(ecs, providerCache)
then:
- amazonloadBalancing.describeTargetHealth({ DescribeTargetHealthRequest request ->
- request.targetGroupArn == targetGroupArn
- request.targets.size() == 1
- request.targets.get(0).id == CommonCachingAgent.EC2_INSTANCE_ID_1
- request.targets.get(0).port == 1338
- }) >> new DescribeTargetHealthResult().withTargetHealthDescriptions(
- new TargetHealthDescription().withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
- )
-
taskHealthList.size() == 1
TaskHealth taskHealth = taskHealthList.get(0)
taskHealth.getState() == 'Up'
@@ -229,15 +228,6 @@ class TaskHealthCachingAgentSpec extends Specification {
def taskHealthList = agent.getItems(ecs, providerCache)
then:
- amazonloadBalancing.describeTargetHealth({ DescribeTargetHealthRequest request ->
- request.targetGroupArn == targetGroupArn
- request.targets.size() == 1
- request.targets.get(0).id == CommonCachingAgent.EC2_INSTANCE_ID_1
- request.targets.get(0).port == 1338
- }) >> new DescribeTargetHealthResult().withTargetHealthDescriptions(
- new TargetHealthDescription().withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
- )
-
taskHealthList.size() == 1
TaskHealth taskHealth = taskHealthList.get(0)
taskHealth.getState() == 'Up'
@@ -303,15 +293,6 @@ class TaskHealthCachingAgentSpec extends Specification {
def taskHealthList = agent.getItems(ecs, providerCache)
then:
- amazonloadBalancing.describeTargetHealth({ DescribeTargetHealthRequest request ->
- request.targetGroupArn == targetGroupArn
- request.targets.size() == 1
- request.targets.get(0).id == "192.168.0.100"
- request.targets.get(0).port == 1338
- }) >> new DescribeTargetHealthResult().withTargetHealthDescriptions(
- new TargetHealthDescription().withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
- )
-
taskHealthList.size() == 1
TaskHealth taskHealth = taskHealthList.get(0)
taskHealth.getState() == 'Up'

feat(ecs): Refactor TaskHealthCachingAgent to use TargetHealthCache (#4275)

* feat(ecs): Refactor TaskHealthCachingAgent to use TargetHealthCache

* Determines Task health with cached ALB target healths vs.
calls to Elastic Load Balancing

* Return status_unknown for targets not found