Merge pull request #364 from CecileRobertMichon/e2e-tests-improvements

Add 3 control plane cluster E2E test

Author: k8s-ci-robot

.gitignore

@@ -49,3 +49,6 @@ bazel-*
 
 # Tilt files.
 .tiltbuild
+
+# junit output
+test/e2e/junit.e2e_suite.*.xml

Makefile

@@ -93,7 +93,7 @@ test-integration: ## Run integration tests
 test-e2e: ## Run e2e tests
 	PULL_POLICY=IfNotPresent $(MAKE) docker-build
 	MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
-	go test ./test/e2e -v -tags=e2e -ginkgo.v -ginkgo.trace -count=1 -timeout=30m
+	go test ./test/e2e -v -tags=e2e -ginkgo.v -ginkgo.trace -count=1 -timeout=60m
 
 $(KUBECTL) $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools

test/e2e/azure_suite_test.go

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_test
+package e2e
 
 import (
 	"bufio"
@@ -54,23 +54,13 @@ func TestE2E(t *testing.T) {
 	junitReporter := reporters.NewJUnitReporter(junitPath)
 
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "CAPZ e2e suite", []Reporter{junitReporter})
+	RunSpecsWithDefaultAndCustomReporters(t, "capz e2e suite", []Reporter{junitReporter})
 }
 
 var (
 	ctx   = context.Background()
 	creds auth.Creds
 	mgmt  *kind.Cluster
-
-	// TODO Parameterize some of these variables
-	location       = "westus2"
-	vmSize         = "Standard_B2ms"
-	namespace      = "default"
-	k8sVersion     = "v1.16.2"
-	imageOffer     = "capi"
-	imagePublisher = "cncf-upstream"
-	imageSKU       = "k8s-1dot16-ubuntu-1804"
-	imageVersion   = "latest"
 )
 
 var _ = BeforeSuite(func() {

test/e2e/azure_test.go

@@ -16,277 +16,70 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e_test
+package e2e
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
-	"encoding/base64"
-	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 
-	"github.com/pkg/errors"
-	"golang.org/x/crypto/ssh"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	bootstrapv1 "sigs.k8s.io/cluster-api-bootstrap-provider-kubeadm/api/v1alpha2"
-	kubeadmv1beta1 "sigs.k8s.io/cluster-api-bootstrap-provider-kubeadm/kubeadm/v1beta1"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2"
 	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/framework"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
-	"sigs.k8s.io/cluster-api/util"
 )
 
 var _ = Describe("CAPZ e2e tests", func() {
 	Describe("Cluster creation", func() {
 
-		clusterGen := &ClusterGenerator{}
-		nodeGen := &NodeGenerator{}
+		var (
+			clusterGen           *ClusterGenerator
+			nodeGen              *NodeGenerator
+			cluster              *capiv1.Cluster
+			infraCluster         *infrav1.AzureCluster
+			machineDeploymentGen = &MachineDeploymentGenerator{}
+		)
+
+		BeforeEach(func() {
+			clusterGen = &ClusterGenerator{}
+			nodeGen = &NodeGenerator{}
+			cluster, infraCluster = clusterGen.GenerateCluster(namespace)
+		})
 
-		Context("Create one controlplane cluster", func() {
-			It("Should create a single node cluster and delete it before exiting", func() {
-				cluster, infraCluster := clusterGen.GenerateCluster(namespace)
-				node := nodeGen.GenerateNode(cluster.GetName())
-				OneNodeCluster(&OneNodeClusterInput{
+		AfterEach(func() {
+			By("cleaning up e2e resources")
+			framework.CleanUp(&framework.CleanUpInput{
+				Management:    mgmt,
+				Cluster:       cluster,
+				DeleteTimeout: 20 * time.Minute,
+			})
+		})
+
+		Context("Create single controlplane cluster", func() {
+			It("Should create a single node cluster", func() {
+				nodes := []framework.Node{nodeGen.GenerateNode(creds, cluster.GetName())}
+				ControlPlaneCluster(&ControlPlaneClusterInput{
 					Management:    mgmt,
 					Cluster:       cluster,
 					InfraCluster:  infraCluster,
-					Node:          node,
-					CreateTimeout: 60 * time.Minute,
-				})
-				framework.CleanUp(&framework.CleanUpInput{
-					Management: mgmt,
-					Cluster:    cluster,
+					Nodes:         nodes,
+					CreateTimeout: 30 * time.Minute,
 				})
 			})
 		})
 
-		// TODO: Deploy multiple Control Plane
-		// TODO: Deploy Addons
-		// TODO: Deploy MachineDeployments
-		// TODO: Scale up
-		// TODO: Scale down
+		Context("Create multiple controlplane cluster with machine deployments", func() {
+			It("Should create a 3 node cluster", func() {
+				nodes := []framework.Node{nodeGen.GenerateNode(creds, cluster.GetName()), nodeGen.GenerateNode(creds, cluster.GetName()), nodeGen.GenerateNode(creds, cluster.GetName())}
+				machineDeployment := machineDeploymentGen.Generate(creds, cluster.GetNamespace(), cluster.GetName(), 1)
+				ControlPlaneCluster(&ControlPlaneClusterInput{
+					Management:        mgmt,
+					Cluster:           cluster,
+					InfraCluster:      infraCluster,
+					Nodes:             nodes,
+					MachineDeployment: machineDeployment,
+					CreateTimeout:     30 * time.Minute,
+				})
+			})
+		})
 	})
 })
-
-type ClusterGenerator struct{}
-
-func (c *ClusterGenerator) GenerateCluster(namespace string) (*capiv1.Cluster, *infrav1.AzureCluster) {
-	name := "capz-" + util.RandomString(6)
-	vnetName := name + "-vnet"
-	infraCluster := &infrav1.AzureCluster{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-		},
-		Spec: infrav1.AzureClusterSpec{
-			Location:      location,
-			ResourceGroup: name,
-			NetworkSpec: infrav1.NetworkSpec{
-				Vnet: infrav1.VnetSpec{Name: vnetName},
-			},
-		},
-	}
-	cluster := &capiv1.Cluster{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-		},
-		Spec: capiv1.ClusterSpec{
-			ClusterNetwork: &capiv1.ClusterNetwork{
-				Pods: &capiv1.NetworkRanges{CIDRBlocks: []string{"192.168.0.0/16"}},
-			},
-			InfrastructureRef: &corev1.ObjectReference{
-				APIVersion: infrav1.GroupVersion.String(),
-				Kind:       framework.TypeToKind(infraCluster),
-				Namespace:  infraCluster.GetNamespace(),
-				Name:       infraCluster.GetName(),
-			},
-		},
-	}
-	return cluster, infraCluster
-}
-
-type NodeGenerator struct {
-	counter int
-}
-
-func (n *NodeGenerator) GenerateNode(clusterName string) framework.Node {
-	sshkey, err := sshkey()
-	Expect(err).NotTo(HaveOccurred())
-
-	firstControlPlane := n.counter == 0
-	name := fmt.Sprintf("%s-controlplane-%d", clusterName, n.counter)
-	n.counter++
-
-	infraMachine := &infrav1.AzureMachine{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-		},
-		Spec: infrav1.AzureMachineSpec{
-			VMSize:       vmSize,
-			Location:     location,
-			SSHPublicKey: sshkey,
-			Image: &infrav1.Image{
-				Offer:     &imageOffer,
-				Publisher: &imagePublisher,
-				SKU:       &imageSKU,
-				Version:   &imageVersion,
-			},
-			OSDisk: infrav1.OSDisk{
-				DiskSizeGB: 30,
-				OSType:     "Linux",
-				ManagedDisk: infrav1.ManagedDisk{
-					StorageAccountType: "Premium_LRS",
-				},
-			},
-		},
-	}
-	bootstrapConfig := &bootstrapv1.KubeadmConfig{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-		},
-		Spec: bootstrapv1.KubeadmConfigSpec{
-			Files: []bootstrapv1.File{
-				{
-					Owner:       "root:root",
-					Path:        "/etc/kubernetes/azure.json",
-					Permissions: "0644",
-					Content:     cloudConfig(clusterName),
-				},
-			},
-			InitConfiguration: &kubeadmv1beta1.InitConfiguration{},
-			JoinConfiguration: &kubeadmv1beta1.JoinConfiguration{},
-		},
-	}
-	registrationOptions := kubeadmv1beta1.NodeRegistrationOptions{
-		Name: "{{ ds.meta_data[\"local_hostname\"] }}",
-		KubeletExtraArgs: map[string]string{
-			"cloud-provider": "azure",
-			"cloud-config":   "/etc/kubernetes/azure.json",
-		},
-	}
-
-	if firstControlPlane {
-		cpInitConfiguration(bootstrapConfig, registrationOptions)
-	} else {
-		cpJoinConfiguration(bootstrapConfig, registrationOptions)
-	}
-
-	machine := &capiv1.Machine{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-			Labels: map[string]string{
-				capiv1.MachineControlPlaneLabelName: "true",
-				capiv1.MachineClusterLabelName:      clusterName,
-			},
-		},
-		Spec: capiv1.MachineSpec{
-			Bootstrap: capiv1.Bootstrap{
-				ConfigRef: &corev1.ObjectReference{
-					APIVersion: bootstrapv1.GroupVersion.String(),
-					Kind:       framework.TypeToKind(bootstrapConfig),
-					Namespace:  bootstrapConfig.GetNamespace(),
-					Name:       bootstrapConfig.GetName(),
-				},
-			},
-			InfrastructureRef: corev1.ObjectReference{
-				APIVersion: infrav1.GroupVersion.String(),
-				Kind:       framework.TypeToKind(infraMachine),
-				Namespace:  infraMachine.GetNamespace(),
-				Name:       infraMachine.GetName(),
-			},
-			Version: &k8sVersion,
-		},
-	}
-	return framework.Node{
-		Machine:         machine,
-		InfraMachine:    infraMachine,
-		BootstrapConfig: bootstrapConfig,
-	}
-}
-
-func cloudConfig(cn string) string {
-	// TODO This is ugly
-	return fmt.Sprintf(`{
-    "cloud": "AzurePublicCloud",
-    "tenantId": "%s",
-    "subscriptionId": "%s",
-    "aadClientId": "%s",
-    "aadClientSecret": "%s",
-    "resourceGroup": "%s",
-    "securityGroupName": "%s-controlplane-nsg",
-    "location": "westus2",
-    "vmType": "standard",
-    "vnetName": "%s-vnet",
-    "vnetResourceGroup": "%s",
-    "subnetName": "%s-controlplane-subnet",
-    "routeTableName": "%s-node-routetable",
-    "userAssignedID": "%s",
-    "loadBalancerSku": "standard",
-    "maximumLoadBalancerRuleCount": 250,
-    "useManagedIdentityExtension": false,
-    "useInstanceMetadata": true
-}`, creds.TenantID, creds.SubscriptionID, creds.ClientID, creds.ClientSecret, cn, cn, cn, cn, cn, cn, cn)
-}
-
-func sshkey() (string, error) {
-	// TODO Load from AZURE_SSH_PUBLIC_KEY_FILE if set
-	prv, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return "", errors.Wrap(err, "Failed to generate private key")
-	}
-	pub, err := ssh.NewPublicKey(&prv.PublicKey)
-	if err != nil {
-		return "", errors.Wrap(err, "Failed to generate public key")
-	}
-	return base64.StdEncoding.EncodeToString(ssh.MarshalAuthorizedKey(pub)), nil
-}
-
-func cpInitConfiguration(kubeadmConfig *bootstrapv1.KubeadmConfig, registrationOptions kubeadmv1beta1.NodeRegistrationOptions) {
-	kubeadmConfig.Spec.ClusterConfiguration = &kubeadmv1beta1.ClusterConfiguration{
-		APIServer: kubeadmv1beta1.APIServer{
-			ControlPlaneComponent: kubeadmv1beta1.ControlPlaneComponent{
-				ExtraArgs: map[string]string{
-					"cloud-provider": "azure",
-					"cloud-config":   "/etc/kubernetes/azure.json",
-				},
-				ExtraVolumes: []kubeadmv1beta1.HostPathMount{
-					{
-						Name:      "cloud-config",
-						HostPath:  "/etc/kubernetes/azure.json",
-						MountPath: "/etc/kubernetes/azure.json",
-						ReadOnly:  true,
-					},
-				},
-			},
-			TimeoutForControlPlane: &metav1.Duration{Duration: 20 * time.Minute},
-		},
-		ControllerManager: kubeadmv1beta1.ControlPlaneComponent{
-			ExtraArgs: map[string]string{
-				"allocate-node-cidrs": "false",
-				"cloud-provider":      "azure",
-				"cloud-config":        "/etc/kubernetes/azure.json",
-			},
-			ExtraVolumes: []kubeadmv1beta1.HostPathMount{
-				{
-					Name:      "cloud-config",
-					HostPath:  "/etc/kubernetes/azure.json",
-					MountPath: "/etc/kubernetes/azure.json",
-					ReadOnly:  true,
-				},
-			},
-		},
-	}
-	kubeadmConfig.Spec.InitConfiguration = &kubeadmv1beta1.InitConfiguration{NodeRegistration: registrationOptions}
-}
-
-func cpJoinConfiguration(kubeadmConfig *bootstrapv1.KubeadmConfig, registrationOptions kubeadmv1beta1.NodeRegistrationOptions) {
-	kubeadmConfig.Spec.JoinConfiguration = &kubeadmv1beta1.JoinConfiguration{NodeRegistration: registrationOptions, ControlPlane: nil}
-}