test: add e2e tests for cert rotation
This adds an e2e test that rotates the certificates then checks:
* if all kubelets are up
* the kubeconfig got changed
* the API serves a longer valid cert
MODIFY
Makefile
Makefile
@@ -101,7 +101,8 @@ run-e2e-tests:
KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -mod=$(MOD) -tags="$(platform),poste2e" -covermode=atomic -buildmode=exe -v -count=1 -timeout=17m ./test/...
# This is a test that should be run in the end to reduce the disruption to other tests because
# it will delete a node.
- KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -mod=$(MOD) -tags="$(platform),disruptivee2e" -covermode=atomic -buildmode=exe -v -count=1 ./test/...
+ # The timeout is made longer as it will perform actions that will take longer than the default 10 minutes.
+ KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -timeout 30m -mod=$(MOD) -tags="$(platform),disruptivee2e" -covermode=atomic -buildmode=exe -v -count=1 ./test/...
.PHONY: all
all: build test

MODIFY
ci/scripts/deploy-cluster.sh
ci/scripts/deploy-cluster.sh
@@ -93,6 +93,7 @@ resource_dir=$(pwd)/..
cd "ci/$platform"
cat "$platform-cluster.lokocfg.envsubst" | envsubst '$AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $PUB_KEY $CLUSTER_ID $AWS_DEFAULT_REGION $AWS_DNS_ZONE $AWS_DNS_ZONE_ID $PACKET_PROJECT_ID $EMAIL $GITHUB_CLIENT_ID $GITHUB_CLIENT_SECRET $DEX_STATIC_CLIENT_CLUSTERAUTH_ID $DEX_STATIC_CLIENT_CLUSTERAUTH_SECRET $GANGWAY_REDIRECT_URL $GANGWAY_SESSION_KEY $DEX_INGRESS_HOST $GANGWAY_INGRESS_HOST $ISSUER_HOST $REDIRECT_URI $API_SERVER_URL $AUTHORIZE_URL $TOKEN_URL $PACKET_LOCATION $ARM_SUBSCRIPTION_ID $ARM_TENANT_ID $ARM_CLIENT_ID $ARM_CLIENT_SECRET' >"$platform-cluster.lokocfg"
+export LOKOCFG_LOCATION="$(pwd)/$platform-cluster.lokocfg"
export KUBECONFIG=$HOME/lokoctl-assets/cluster-assets/auth/kubeconfig
echo "export KUBECONFIG=$KUBECONFIG" >> ~/.bashrc

ADD
test/components/kubernetes/cert_rotate_disruptive_test.go
+// Copyright 2021 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build packet baremetal aws
+// +build disruptivee2e
+
+package kubernetes_test
+
+import (
+ "bytes"
+ "crypto/tls"
+ "crypto/x509"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "testing"
+
+ log "github.com/sirupsen/logrus"
+
+ "github.com/kinvolk/lokomotive/cli/cmd/cluster"
+ testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+func TestCertificateRotate(t *testing.T) { //nolint:funlen
+ contextLogger := log.WithFields(log.Fields{
+ "command": "TestCertificateRotate",
+ })
+
+ oldConfig := testutil.BuildKubeConfig(t)
+
+ oldCerts := fetchCertificatesForEndpoint(t, oldConfig.Host)
+
+ cfgPath := testutil.LokocfgPath(t)
+ dir, _ := path.Split(cfgPath)
+
+ // We will be changing directories for relative paths in lokocfg to function.
+ // This will revert it back when this test exits.
+ testDir, err := os.Getwd()
+ if err != nil {
+ t.Fatalf("Getting current work directory failed: %v", err)
+ }
+
+ t.Cleanup(func() {
+ if err := os.Chdir(testDir); err != nil {
+ t.Fatalf("Failed to change back %q: %v", testDir, err)
+ }
+ })
+
+ if err := os.Chdir(dir); err != nil {
+ t.Fatalf("Changing work directory failed: %v", err)
+ }
+
+ options := cluster.CertificateRotateOptions{
+ Confirm: true,
+ Verbose: true,
+ ConfigPath: cfgPath,
+ ValuesPath: path.Join(dir, "lokocfg.vars"),
+ }
+
+ if err := cluster.RotateCertificates(contextLogger, options); err != nil {
+ t.Fatalf("Rotating Certificates failed: %v", err)
+ }
+
+ newConfig := testutil.BuildKubeConfig(t)
+
+ if bytes.Equal(oldConfig.CAData, newConfig.CAData) {
+ t.Fatal("CA Certificate is identical after rotation")
+ }
+
+ client := testutil.CreateKubeClient(t)
+
+ // Test if all kubelets are running.
+ namespace := "kube-system"
+ daemonset := "kubelet"
+
+ t.Run("leaves_all_self_hosted_kubelets_running", func(t *testing.T) {
+ testutil.WaitForDaemonSet(t, client, namespace, daemonset, testutil.RetryInterval, testutil.Timeout)
+
+ newCerts := fetchCertificatesForEndpoint(t, oldConfig.Host)
+
+ // Currently Lokomotive doesn't use intermediate CAs, thus we can safely assume the first one is the leaf cert.
+ if oldCerts[0].NotAfter.After(newCerts[0].NotAfter) || oldCerts[0].NotAfter.Equal(newCerts[0].NotAfter) {
+ t.Fatalf("Incorrect not after time on new certificate: %v is equal or before %v",
+ newCerts[0].NotAfter, oldCerts[0].NotAfter)
+ }
+ })
+}
+
+func fetchCertificatesForEndpoint(t *testing.T, url string) []*x509.Certificate {
+ if !strings.HasPrefix(url, "https://") {
+ url = "https://" + url
+ }
+
+ /* #nosec G402*/
+ tr := &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ client := &http.Client{Transport: tr}
+
+ resp, err := client.Get(url) //nolint:noctx
+ if err != nil {
+ t.Fatalf("Getting url %q: %v", url, err)
+ }
+
+ t.Cleanup(func() {
+ if err := resp.Body.Close(); err != nil {
+ t.Fatalf("Failed to close response body: %v", err)
+ }
+ })
+
+ if len(resp.TLS.PeerCertificates) < 1 {
+ t.Fatalf("No certificates offered in TLS handshake")
+ }
+
+ return resp.TLS.PeerCertificates
+}

MODIFY
test/components/util/util.go
test/components/util/util.go
@@ -87,10 +87,10 @@ func Kubeconfig(t *testing.T) []byte {
return k
}
-// buildKubeConfig reads the environment variable KUBECONFIG and then builds the rest client config
+// BuildKubeConfig reads the environment variable KUBECONFIG and then builds the rest client config
// object which can be either used to create kube client to talk to apiserver or to just read the
// kubeconfig data.
-func buildKubeConfig(t *testing.T) *restclient.Config {
+func BuildKubeConfig(t *testing.T) *restclient.Config {
kubeconfig := KubeconfigPath(t)
t.Logf("using KUBECONFIG=%s", kubeconfig)
@@ -105,7 +105,7 @@ func buildKubeConfig(t *testing.T) *restclient.Config {
// CreateKubeClient returns a kubernetes client reading the KUBECONFIG environment variable.
func CreateKubeClient(t *testing.T) *kubernetes.Clientset {
- cs, err := kubernetes.NewForConfig(buildKubeConfig(t))
+ cs, err := kubernetes.NewForConfig(BuildKubeConfig(t))
if err != nil {
t.Fatalf("failed creating new clientset: %v", err)
}
@@ -392,7 +392,7 @@ func (p *PortForwardInfo) CloseChan() {
// p.WaitUntilForwardingAvailable(t)
//
func (p *PortForwardInfo) PortForward(t *testing.T) {
- config := buildKubeConfig(t)
+ config := BuildKubeConfig(t)
roundTripper, upgrader, err := spdy.RoundTripperFor(config)
if err != nil {

test: add e2e tests for cert rotation

This adds an e2e test that rotates the certificates then checks:
* if all kubelets are up
* the kubeconfig got changed
* the API serves a longer valid cert

Signed-off-by: Maartje Eyskens <maartje@kinvolk.io>