fix(cf): reduce unnecessary calls to foundation to improve performance.
MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/Applications.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/Applications.java
@@ -96,17 +96,47 @@ public class Applications {
}
public List<CloudFoundryApplication> all() {
+ List<Application> newCloudFoundryAppList =
+ collectPages("applications", page -> api.all(page, 5000, null, null));
+
+ // Evict Records from `cache` that are no longer in the foundation
+ List<String> availableAppIds =
+ newCloudFoundryAppList.stream().map(Application::getGuid).collect(toList());
+ serverGroupCache.asMap().keySet().stream()
+ .forEach(
+ key -> {
+ if (!availableAppIds.contains(key)) {
+ log.debug("Evicting the following SG with id '" + key + "'");
+ serverGroupCache.invalidate(key);
+ }
+ });
+
+ // if the update time doesn't match then we need to update the cache
+ // if the app is not found in the cache we need to process with `map` and update the cache
+ List<Application> appsToBeUpdated =
+ newCloudFoundryAppList.stream()
+ .filter(
+ a ->
+ Optional.ofNullable(findById(a.getGuid()))
+ .map(
+ r ->
+ !r.getUpdatedTime()
+ .equals(a.getUpdatedAt().toInstant().toEpochMilli()))
+ .orElse(true))
+ .collect(Collectors.toList());
+
List<CloudFoundryServerGroup> serverGroups =
- collectPages("applications", page -> api.all(page, null, null)).stream()
- .map(this::map)
- .collect(toList());
- serverGroupCache.invalidateAll();
+ appsToBeUpdated.stream().map(this::map).collect(toList());
serverGroups.forEach(sg -> serverGroupCache.put(sg.getId(), sg));
+ // execute health check on instances, set number of available instances and health status
+ newCloudFoundryAppList.forEach(
+ a -> serverGroupCache.put(a.getGuid(), checkHealthStatus(findById(a.getGuid()), a)));
+
Map<String, Set<CloudFoundryServerGroup>> serverGroupsByClusters = new HashMap<>();
Map<String, Set<String>> clustersByApps = new HashMap<>();
- for (CloudFoundryServerGroup serverGroup : serverGroups) {
+ for (CloudFoundryServerGroup serverGroup : serverGroupCache.asMap().values()) {
Names names = Names.parseName(serverGroup.getName());
serverGroupsByClusters
.computeIfAbsent(names.getCluster(), clusterName -> new HashSet<>())
@@ -153,7 +183,7 @@ public class Applications {
.map(CloudFoundryServerGroup::getId)
.orElseGet(
() ->
- safelyCall(() -> api.all(null, singletonList(name), singletonList(spaceId)))
+ safelyCall(() -> api.all(null, 1, singletonList(name), singletonList(spaceId)))
.flatMap(
page ->
page.getResources().stream()
@@ -180,68 +210,6 @@ public class Applications {
safelyCall(() -> api.findApplicationEnvById(appId)).orElse(null);
Process process = safelyCall(() -> api.findProcessById(appId)).orElse(null);
- Set<CloudFoundryInstance> instances;
- switch (state) {
- case STOPPED:
- instances = emptySet();
- break;
- case STARTED:
- try {
- instances =
- safelyCall(() -> api.instances(appId)).orElse(emptyMap()).entrySet().stream()
- .map(
- inst -> {
- HealthState healthState = HealthState.Unknown;
- switch (inst.getValue().getState()) {
- case RUNNING:
- healthState = HealthState.Up;
- break;
- case DOWN:
- case CRASHED:
- healthState = HealthState.Down;
- break;
- case STARTING:
- healthState = HealthState.Starting;
- break;
- }
- return CloudFoundryInstance.builder()
- .appGuid(appId)
- .key(inst.getKey())
- .healthState(healthState)
- .details(inst.getValue().getDetails())
- .launchTime(
- System.currentTimeMillis() - (inst.getValue().getUptime() * 1000))
- .zone(space == null ? "unknown" : space.getName())
- .build();
- })
- .collect(toSet());
-
- log.debug(
- "Successfully retrieved "
- + instances.size()
- + " instances for application '"
- + application.getName()
- + "'");
- } catch (RetrofitError e) {
- try {
- log.debug(
- "Unable to retrieve instances for application '"
- + application.getName()
- + "': "
- + IOUtils.toString(e.getResponse().getBody().in(), Charset.defaultCharset()));
- } catch (IOException e1) {
- log.debug("Unable to retrieve droplet for application '" + application.getName() + "'");
- }
- instances = emptySet();
- } catch (Exception ex) {
- log.debug("Unable to retrieve droplet for application '" + application.getName() + "'");
- instances = emptySet();
- }
- break;
- default:
- instances = emptySet();
- }
-
CloudFoundryDroplet droplet = null;
try {
CloudFoundryPackage cfPackage =
@@ -350,7 +318,8 @@ public class Applications {
serverGroupMetricsUri = metricsUri + "/apps/" + appId;
}
- return CloudFoundryServerGroup.builder()
+ CloudFoundryServerGroup cloudFoundryServerGroup =
+ CloudFoundryServerGroup.builder()
.account(account)
.appsManagerUri(serverGroupAppManagerUri)
.metricsUri(serverGroupMetricsUri)
@@ -365,13 +334,82 @@ public class Applications {
.space(space)
.createdTime(application.getCreatedAt().toInstant().toEpochMilli())
.serviceInstances(cloudFoundryServices)
- .instances(instances)
.state(state)
.env(environmentVars)
.ciBuild(buildInfo)
.appArtifact(artifactInfo)
.pipelineId(pipelineId)
+ .updatedTime(application.getUpdatedAt().toInstant().toEpochMilli())
.build();
+
+ return checkHealthStatus(cloudFoundryServerGroup, application);
+ }
+
+ private CloudFoundryServerGroup checkHealthStatus(
+ CloudFoundryServerGroup cloudFoundryServerGroup, Application application) {
+ CloudFoundryServerGroup.State state =
+ CloudFoundryServerGroup.State.valueOf(application.getState());
+ Set<CloudFoundryInstance> instances;
+ switch (state) {
+ case STARTED:
+ try {
+ instances =
+ safelyCall(() -> api.instances(cloudFoundryServerGroup.getId())).orElse(emptyMap())
+ .entrySet().stream()
+ .map(
+ inst -> {
+ HealthState healthState = HealthState.Unknown;
+ switch (inst.getValue().getState()) {
+ case RUNNING:
+ healthState = HealthState.Up;
+ break;
+ case DOWN:
+ case CRASHED:
+ healthState = HealthState.Down;
+ break;
+ case STARTING:
+ healthState = HealthState.Starting;
+ break;
+ }
+ return CloudFoundryInstance.builder()
+ .appGuid(cloudFoundryServerGroup.getId())
+ .key(inst.getKey())
+ .healthState(healthState)
+ .details(inst.getValue().getDetails())
+ .launchTime(
+ System.currentTimeMillis() - (inst.getValue().getUptime() * 1000))
+ .zone(cloudFoundryServerGroup.getRegion())
+ .build();
+ })
+ .collect(toSet());
+
+ log.debug(
+ "Successfully retrieved "
+ + instances.size()
+ + " instances for application '"
+ + application.getName()
+ + "'");
+ } catch (RetrofitError e) {
+ try {
+ log.debug(
+ "Unable to retrieve instances for application '"
+ + application.getName()
+ + "': "
+ + IOUtils.toString(e.getResponse().getBody().in(), Charset.defaultCharset()));
+ } catch (IOException e1) {
+ log.debug("Unable to retrieve droplet for application '" + application.getName() + "'");
+ }
+ instances = emptySet();
+ } catch (Exception ex) {
+ log.debug("Unable to retrieve droplet for application '" + application.getName() + "'");
+ instances = emptySet();
+ }
+ break;
+ case STOPPED:
+ default:
+ instances = emptySet();
+ }
+ return cloudFoundryServerGroup.toBuilder().state(state).instances(instances).build();
}
private String getEnvironmentVar(

MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/Routes.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/Routes.java
@@ -59,7 +59,7 @@ public class Routes {
private LoadingCache<String, List<RouteMapping>> routeMappings =
CacheBuilder.newBuilder()
- .expireAfterWrite(1, TimeUnit.SECONDS)
+ .expireAfterWrite(30, TimeUnit.SECONDS)
.build(
new CacheLoader<String, List<RouteMapping>>() {
@Override
@@ -111,7 +111,7 @@ public class Routes {
if (routeId.getPath() != null) queryParams.add("path:" + routeId.getPath());
if (routeId.getPort() != null) queryParams.add("port:" + routeId.getPort().toString());
- return collectPageResources("route mappings", pg -> api.all(pg, queryParams)).stream()
+ return collectPageResources("route mappings", pg -> api.all(pg, 5000, queryParams)).stream()
.filter(
routeResource ->
(routeId.getPath() != null || routeResource.getEntity().getPath().isEmpty())
@@ -142,7 +142,8 @@ public class Routes {
}
public List<CloudFoundryLoadBalancer> all() throws CloudFoundryApiException {
- List<Resource<Route>> routeResources = collectPageResources("routes", pg -> api.all(pg, null));
+ List<Resource<Route>> routeResources =
+ collectPageResources("routes", pg -> api.all(pg, 5000, null));
List<CloudFoundryLoadBalancer> loadBalancers = new ArrayList<>(routeResources.size());
for (Resource<Route> routeResource : routeResources) {
loadBalancers.add(map(routeResource));

MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/ApplicationService.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/ApplicationService.java
@@ -33,6 +33,7 @@ public interface ApplicationService {
@GET("/v3/apps")
Pagination<Application> all(
@Query("page") Integer page,
+ @Query("per_page") Integer perPage,
@Query("names") List<String> names,
@Query("space_guids") List<String> spaceGuids);

MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/RouteService.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/RouteService.java
@@ -28,7 +28,10 @@ public interface RouteService {
// Mapping to CF API style query params -
// https://apidocs.cloudfoundry.org/1.34.0/routes/list_all_routes.html
@GET("/v2/routes?results-per-page=100")
- Page<Route> all(@Query("page") Integer page, @Query("q") List<String> queryParams);
+ Page<Route> all(
+ @Query("page") Integer page,
+ @Query("per_page") Integer perPage,
+ @Query("q") List<String> queryParams);
@GET("/v2/routes/{guid}")
Resource<Route> findById(@Path("guid") String guid);

MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v3/Application.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v3/Application.java
@@ -26,5 +26,6 @@ public class Application {
private String guid;
private String state;
private ZonedDateTime createdAt;
+ private ZonedDateTime updatedAt;
private Map<String, Link> links;
}

MODIFY
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/model/CloudFoundryServerGroup.java
clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/model/CloudFoundryServerGroup.java
@@ -43,7 +43,7 @@ import lombok.experimental.Wither;
@Value
@EqualsAndHashCode(of = "id", callSuper = false)
-@Builder
+@Builder(toBuilder = true)
@JsonDeserialize(builder = CloudFoundryServerGroup.CloudFoundryServerGroupBuilder.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
@JsonIgnoreProperties("loadBalancerNames")
@@ -89,6 +89,9 @@ public class CloudFoundryServerGroup extends CloudFoundryModel implements Server
@JsonView(Views.Cache.class)
CloudFoundrySpace space;
+ @JsonView(Views.Cache.class)
+ Long updatedTime;
+
@JsonView(Views.Cache.class)
Long createdTime;

MODIFY
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/ApplicationsTest.java
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/ApplicationsTest.java
@@ -97,6 +97,7 @@ class ApplicationsTest {
Application application =
new Application()
.setCreatedAt(ZonedDateTime.now())
+ .setUpdatedAt(ZonedDateTime.now())
.setGuid(serverGroupId)
.setName(serverGroupName)
.setState("STARTED")
@@ -321,6 +322,7 @@ class ApplicationsTest {
Application application =
new Application()
.setCreatedAt(ZonedDateTime.now())
+ .setUpdatedAt(ZonedDateTime.now())
.setGuid(expectedServerGroupId)
.setName("app")
.setState("STARTED")
@@ -331,7 +333,7 @@ class ApplicationsTest {
new Pagination<Application>()
.setPagination(new Pagination.Details().setTotalPages(1))
.setResources(Collections.singletonList(application));
- when(applicationService.all(any(), any(), any())).thenReturn(applicationPagination);
+ when(applicationService.all(any(), any(), any(), any())).thenReturn(applicationPagination);
mockMap(cloudFoundrySpace, "droplet-id");
String serverGroupId = apps.findServerGroupId(serverGroupName, spaceId);
@@ -346,6 +348,7 @@ class ApplicationsTest {
Application application =
new Application()
.setCreatedAt(ZonedDateTime.now())
+ .setUpdatedAt(ZonedDateTime.now())
.setGuid(serverGroupId)
.setName(serverGroupName)
.setState("STARTED")
@@ -364,7 +367,7 @@ class ApplicationsTest {
.setName("service-instance");
String dropletId = "droplet-guid";
- when(applicationService.all(any(), any(), any())).thenReturn(applicationPagination);
+ when(applicationService.all(any(), any(), any(), any())).thenReturn(applicationPagination);
mockMap(cloudFoundrySpace, dropletId);
CloudFoundryDroplet expectedDroplet = CloudFoundryDroplet.builder().id(dropletId).build();
@@ -379,6 +382,7 @@ class ApplicationsTest {
.instances(Collections.emptySet())
.serviceInstances(Collections.emptyList())
.createdTime(application.getCreatedAt().toInstant().toEpochMilli())
+ .updatedTime(application.getUpdatedAt().toInstant().toEpochMilli())
.memory(0)
.diskQuota(0)
.name(serverGroupName)

MODIFY
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/CloudFoundryClientUtilsTest.java
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/CloudFoundryClientUtilsTest.java
@@ -41,11 +41,11 @@ class CloudFoundryClientUtilsTest {
pageOne.setPagination(new Pagination.Details().setTotalPages(1));
pageOne.setResources(pageOneResources);
- when(applicationService.all(null, null, null)).thenReturn(pageOne);
+ when(applicationService.all(null, null, null, null)).thenReturn(pageOne);
List results =
CloudFoundryClientUtils.collectPages(
- "applications", page -> applicationService.all(page, null, null));
+ "applications", page -> applicationService.all(page, null, null, null));
assertThat(results).containsExactly(applicationOne);
}
@@ -64,12 +64,12 @@ class CloudFoundryClientUtilsTest {
pageTwo.setPagination(new Pagination.Details().setTotalPages(2));
pageTwo.setResources(pageTwoResources);
- when(applicationService.all(null, null, null)).thenReturn(pageOne);
- when(applicationService.all(2, null, null)).thenReturn(pageTwo);
+ when(applicationService.all(null, null, null, null)).thenReturn(pageOne);
+ when(applicationService.all(2, null, null, null)).thenReturn(pageTwo);
List results =
CloudFoundryClientUtils.collectPages(
- "applications", page -> applicationService.all(page, null, null));
+ "applications", page -> applicationService.all(page, null, null, null));
assertThat(results).containsExactly(applicationOne, applicationTwo);
}

MODIFY
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/RoutesTest.java
clouddriver-cloudfoundry/src/test/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/RoutesTest.java
@@ -54,7 +54,7 @@ class RoutesTest {
route.setPath("/path");
RouteService routeService = mock(RouteService.class);
- when(routeService.all(any(), any())).thenReturn(Page.singleton(route, "abc123"));
+ when(routeService.all(any(), any(), any())).thenReturn(Page.singleton(route, "abc123"));
when(routeService.routeMappings(any(), any())).thenReturn(new Page<>());
Routes routes = new Routes("pws", routeService, null, domains, spaces);
@@ -124,7 +124,7 @@ class RoutesTest {
routeMappingPage.setTotalPages(1);
when(spaces.findById("space-guid")).thenReturn(space);
- when(routeService.all(any(), any())).thenReturn(routePage);
+ when(routeService.all(any(), any(), any())).thenReturn(routePage);
when(routeService.routeMappings(any(), any())).thenReturn(routeMappingPage);
Routes routes = new Routes("pws", routeService, null, domains, spaces);

fix(cf): reduce unnecessary calls to foundation to improve performance. (#4130)