fix(provider/kubernetes): v2 Check status of workloads after update
MODIFY
clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/model/Manifest.java
clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/model/Manifest.java
@@ -36,6 +36,13 @@ public interface Manifest {
Condition available = Condition.builder().state(true).build();
Condition failed = Condition.builder().state(false).build();
+ public Status unknown() {
+ stable = null;
+ failed = null;
+
+ return this;
+ }
+
public Status failed(String message) {
failed.setMessage(message);
failed.setState(true);

MODIFY
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/manifest/KubernetesManifest.java
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/manifest/KubernetesManifest.java
@@ -163,6 +163,17 @@ public class KubernetesManifest extends HashMap<String, Object> {
return String.join(" ", kind.toString(), name);
}
+ public boolean isNewerThanObservedGeneration() {
+ try {
+ Long generation = (Long) getMetadata().get("generation");
+ Long observedGeneration = ((Map<String, Long>) getStatus()).get("observedGeneration");
+ if (observedGeneration == null || (generation != null && generation > observedGeneration)) {
+ return false;
+ }
+ } catch (ClassCastException e) {
+ }
+ return true;
+ }
/*
* The reasoning behind removing metadata for comparison is that it shouldn't affect the runtime behavior
* of the resource we are creating.

MODIFY
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesDaemonSetHandler.java
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesDaemonSetHandler.java
@@ -79,6 +79,9 @@ public class KubernetesDaemonSetHandler extends KubernetesHandler implements
@Override
public Status status(KubernetesManifest manifest) {
+ if (!manifest.isNewerThanObservedGeneration()) {
+ return (new Status()).unknown();
+ }
V1beta2DaemonSet v1beta2DaemonSet = KubernetesCacheDataConverter.getResource(manifest, V1beta2DaemonSet.class);
return status(v1beta2DaemonSet);
}
@@ -101,6 +104,10 @@ public class KubernetesDaemonSetHandler extends KubernetesHandler implements
return result;
}
+ if (!daemonSet.getSpec().getUpdateStrategy().getType().equalsIgnoreCase("rollingupdate")) {
+ return result;
+ }
+
int desiredReplicas = status.getDesiredNumberScheduled();
Integer existing = status.getCurrentNumberScheduled();
if (existing == null || desiredReplicas > existing) {

MODIFY
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesDeploymentHandler.java
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesDeploymentHandler.java
@@ -78,6 +78,9 @@ public class KubernetesDeploymentHandler extends KubernetesHandler implements
if (manifest.getApiVersion().equals(EXTENSIONS_V1BETA1)
|| manifest.getApiVersion().equals(APPS_V1BETA1)
|| manifest.getApiVersion().equals(APPS_V1BETA2)) {
+ if (!manifest.isNewerThanObservedGeneration()) {
+ return (new Status()).unknown();
+ }
V1beta2Deployment appsV1beta2Deployment = KubernetesCacheDataConverter.getResource(manifest, V1beta2Deployment.class);
return status(appsV1beta2Deployment);
} else {
@@ -119,14 +122,26 @@ public class KubernetesDeploymentHandler extends KubernetesHandler implements
result.unavailable(available.getMessage());
}
- int desiredReplicas = deployment.getSpec().getReplicas();
+ V1beta2DeploymentCondition condition = status.getConditions()
+ .stream()
+ .filter(c -> c.getType().equalsIgnoreCase("progressing"))
+ .findAny()
+ .orElse(null);
+ if (condition != null && condition.getReason().equalsIgnoreCase("progressdeadlineexceeded")) {
+ return result.failed("Deployment exceeded its progress deadline");
+ }
+
+ Integer desiredReplicas = deployment.getSpec().getReplicas();
Integer existing = status.getUpdatedReplicas();
- if (existing == null || desiredReplicas > existing) {
+ if (existing == null || (desiredReplicas != null && desiredReplicas > existing)) {
return result.unstable("Waiting for all replicas to be updated");
}
- existing = status.getAvailableReplicas();
- if (existing == null || desiredReplicas > existing) {
+ if (status.getReplicas() > existing) {
+ return result.unstable("Waiting for old replicas to finish termination");
+ }
+
+ if (status.getAvailableReplicas() < existing) {
return result.unstable("Waiting for all replicas to be available");
}

MODIFY
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesReplicaSetHandler.java
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesReplicaSetHandler.java
@@ -98,6 +98,11 @@ public class KubernetesReplicaSetHandler extends KubernetesHandler implements
return result;
}
+ Long observedGeneration = status.getObservedGeneration();
+ if (observedGeneration != null && observedGeneration != replicaSet.getMetadata().getGeneration()) {
+ result.unstable("Waiting for replicaset spec update to be observed");
+ }
+
Integer existing = status.getFullyLabeledReplicas();
if (existing == null || desiredReplicas > existing) {
return result.unstable("Waiting for all replicas to be fully-labeled")

MODIFY
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesStatefulSetHandler.java
clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesStatefulSetHandler.java
@@ -27,6 +27,7 @@ import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpi
import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
import com.netflix.spinnaker.clouddriver.model.Manifest.Status;
+import io.kubernetes.client.models.V1beta2RollingUpdateStatefulSetStrategy;
import io.kubernetes.client.models.V1beta2StatefulSet;
import io.kubernetes.client.models.V1beta2StatefulSetStatus;
import org.springframework.stereotype.Component;
@@ -80,6 +81,9 @@ public class KubernetesStatefulSetHandler extends KubernetesHandler implements
@Override
public Status status(KubernetesManifest manifest) {
+ if (!manifest.isNewerThanObservedGeneration()) {
+ return (new Status()).unknown();
+ }
V1beta2StatefulSet v1beta2StatefulSet = KubernetesCacheDataConverter.getResource(manifest, V1beta2StatefulSet.class);
return status(v1beta2StatefulSet);
}
@@ -101,6 +105,10 @@ public class KubernetesStatefulSetHandler extends KubernetesHandler implements
private Status status(V1beta2StatefulSet statefulSet) {
Status result = new Status();
+ if (statefulSet.getSpec().getUpdateStrategy().getType().equalsIgnoreCase("ondelete")) {
+ return result;
+ }
+
V1beta2StatefulSetStatus status = statefulSet.getStatus();
if (status == null) {
result.unstable("No status reported yet")
@@ -108,9 +116,9 @@ public class KubernetesStatefulSetHandler extends KubernetesHandler implements
return result;
}
- int desiredReplicas = statefulSet.getSpec().getReplicas();
+ Integer desiredReplicas = statefulSet.getSpec().getReplicas();
Integer existing = status.getReplicas();
- if (existing == null || desiredReplicas > existing) {
+ if (existing == null || (desiredReplicas != null && desiredReplicas > existing)) {
return result.unstable("Waiting for at least the desired replica count to be met");
}
@@ -119,12 +127,12 @@ public class KubernetesStatefulSetHandler extends KubernetesHandler implements
}
existing = status.getCurrentReplicas();
- if (existing == null || desiredReplicas > existing) {
+ if (existing == null || (desiredReplicas != null && desiredReplicas > existing)) {
return result.unstable("Waiting for all updated replicas to be scheduled");
}
existing = status.getReadyReplicas();
- if (existing == null || desiredReplicas > existing) {
+ if (existing == null || (desiredReplicas != null && desiredReplicas > existing)) {
return result.unstable("Waiting for all updated replicas to be ready");
}

fix(provider/kubernetes): v2 Check status of workloads after update (#2515)