add graceful operator shutdown
MODIFY
cmd/manager/main.go
cmd/manager/main.go
@@ -8,21 +8,21 @@ import (
"runtime"
"strings"
- _ "github.com/Percona-Lab/percona-version-service/api"
certmgrscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme"
"github.com/operator-framework/operator-sdk/pkg/k8sutil"
"github.com/operator-framework/operator-sdk/pkg/leader"
"github.com/operator-framework/operator-sdk/pkg/ready"
sdkVersion "github.com/operator-framework/operator-sdk/version"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+ "sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/manager"
- "sigs.k8s.io/controller-runtime/pkg/cache"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
- "sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+ _ "github.com/Percona-Lab/percona-version-service/api"
"github.com/percona/percona-xtradb-cluster-operator/pkg/apis"
"github.com/percona/percona-xtradb-cluster-operator/pkg/controller"
+ "github.com/percona/percona-xtradb-cluster-operator/pkg/k8s"
"github.com/percona/percona-xtradb-cluster-operator/pkg/webhook"
"github.com/percona/percona-xtradb-cluster-operator/version"
)
@@ -127,8 +127,10 @@ func main() {
log.Info("Starting the Cmd.")
+ stopCH := k8s.StartStopSignalHandler(mgr.GetClient(), strings.Split(namespace, ","))
+
// Start the Cmd
- if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
+ if err := mgr.Start(stopCH); err != nil {
log.Error(err, "manager exited non-zero")
os.Exit(1)
}

MODIFY
e2e-tests/operator-self-healing/run
e2e-tests/operator-self-healing/run
@@ -115,7 +115,7 @@ main() {
wait_for_delete pod/$cluster-pxc-3
desc 'stop operator'
- pumba_pod stop
+ pumba_pod "stop -t 10"
desc 'check if operator works fine: scale up from 3 to 4'
cat_config "$test_dir/conf/$cluster.yml" \
| sed -e 's/size: 3/size: 4/' \

ADD
pkg/k8s/graceful_shutdown.go
+package k8s
+
+import (
+ "context"
+ "time"
+
+ api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
+ "github.com/pkg/errors"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/manager/signals"
+)
+
+var log = logf.Log
+
+// StartStopSignalHandler starts gorutine which is waiting for
+// termination signal and returns chan for indication when operator
+// can really stop.
+func StartStopSignalHandler(client client.Client, namespaces []string) <-chan struct{} {
+ stopCH := make(chan struct{})
+ go handleStopSignal(client, namespaces, stopCH)
+ return stopCH
+}
+
+func handleStopSignal(client client.Client, namespaces []string, stopCH chan struct{}) {
+ <-signals.SetupSignalHandler()
+ stop(client, namespaces)
+ close(stopCH)
+}
+
+// Stop is used to understand, when we need to stop operator(usially SIGTERM)
+// to start cleanup process and delete required pxc clusters in current(operator)
+// namespace. See K8SPXC-529
+func stop(cl client.Client, namespaces []string) {
+ log.Info("Got stop signal, starting to list clusters")
+
+ readyToDelete := false
+
+ for !readyToDelete {
+ time.Sleep(5 * time.Second)
+ ready, err := checkClusters(cl, namespaces)
+ if err != nil {
+ log.Error(err, "delete clusters")
+ }
+ readyToDelete = ready
+ }
+}
+
+func checkClusters(cl client.Client, namespaces []string) (bool, error) {
+ for _, ns := range namespaces {
+
+ clusterList := &api.PerconaXtraDBClusterList{}
+
+ err := cl.List(context.TODO(), clusterList, &client.ListOptions{
+ Namespace: ns,
+ })
+ if err != nil {
+ if k8serrors.IsNotFound(err) {
+ continue
+ }
+ return false, errors.Wrapf(err, "list clusters in namespace: %s", ns)
+ }
+
+ if !isClustersReadyToDelete(clusterList.Items) {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+func isClustersReadyToDelete(list []api.PerconaXtraDBCluster) bool {
+ for _, v := range list {
+ if v.ObjectMeta.DeletionTimestamp != nil && len(v.Finalizers) != 0 {
+ return false
+ }
+ }
+ return true
+}

K8SPXC-529: add graceful operator shutdown (#689)