Merge pull request #2764 from Nordix/Sunnatillo/add-failure-domain

 ✨ Add FailureDomain Support

Author: metal3-io-bot

api/v1beta1/metal3cluster_types.go

@@ -66,8 +66,15 @@ type Metal3ClusterSpec struct {
 	// Default value is true, it is set in the webhook.
 	// +optional
 	CloudProviderEnabled *bool `json:"cloudProviderEnabled,omitempty"`
+
+	// FailureDomains specifies a list fo failure zones that can be used
+	// +optional
+	FailureDomains FailureDomains `json:"failureDomains,omitempty"`
 }
 
+// FailureDomains is a slice of FailureDomainSpecs.
+type FailureDomains clusterv1beta1.FailureDomains
+
 // IsValid returns an error if the object is not valid, otherwise nil. The
 // string representation of the error is suitable for human consumption.
 func (s *Metal3ClusterSpec) IsValid() error {
@@ -115,6 +122,9 @@ type Metal3ClusterStatus struct {
 	// v1beta2 groups all the fields that will be added or modified in Metal3Cluster's status with the V1Beta2 version.
 	// +optional
 	V1Beta2 *Metal3ClusterV1Beta2Status `json:"v1beta2,omitempty"`
+	// FailureDomains specifies a list fo failure zones that can be used
+	// +optional
+	FailureDomains FailureDomains `json:"failureDomains,omitempty"`
 }
 
 // Metal3ClusterV1Beta2Status groups all the fields that will be added or modified in Metal3ClusterStatus with the V1Beta2 version.

api/v1beta1/metal3machine_types.go

@@ -160,6 +160,9 @@ type Metal3MachineSpec struct {
 	// +kubebuilder:validation:Enum:=metadata;disabled
 	// +optional
 	AutomatedCleaningMode *string `json:"automatedCleaningMode,omitempty"`
+
+	// FailureDomain is the failure domain unique identifier this Machine should be attached to, as defined in Cluster API.
+	FailureDomain string `json:"failureDomain,omitempty"`
 }
 
 // Metal3MachineStatus defines the observed state of Metal3Machine.

api/v1beta1/zz_generated.deepcopy.go

@@ -76,6 +76,8 @@ const (
 	ProviderIDPrefix = "metal3://"
 	// ProviderLabelPrefix is a label prefix for ProviderID.
 	ProviderLabelPrefix = "metal3.io/uuid"
+	// FailureDomainLabelName is a label name for FailureDomains.
+	FailureDomainLabelName = "infrastructure.cluster.x-k8s.io/failure-domain"
 )
 
 var (
@@ -924,9 +926,11 @@ func (m *MachineManager) chooseHost(ctx context.Context) (*bmov1alpha1.BareMetal
 			// If host is found in `Ready` state, pick it
 			if len(hostsInAvailableStateWithNodeReuse) != 0 {
 				m.Log.Info("Found host(s) with nodeReuseLabelName in Ready/Available state, choosing the host", "availabeHostCount", len(hostsInAvailableStateWithNodeReuse), "host", host.Name)
-				rHost, _ := rand.Int(rand.Reader, big.NewInt(int64(len(hostsInAvailableStateWithNodeReuse))))
-				randomHost := rHost.Int64()
-				chosenHost = hostsInAvailableStateWithNodeReuse[randomHost]
+				chosenHost, err = m.pickHost(hostsInAvailableStateWithNodeReuse)
+				if err != nil {
+					m.Log.Error(err, "Failed to choose host, not choosing host")
+					return nil, nil, err
+				}
 			} else if len(hostsInNotAvailableStateWithNodeReuse) != 0 {
 				errMessage := fmt.Sprint("Found BareMetalHost(s) with nodeReuseLabelName in not-available state, requeuing the BareMetalHost", "notAvailabeHostCount", len(hostsInNotAvailableStateWithNodeReuse), "hoststate", host.Status.Provisioning.State, "host", host.Name)
 				m.Log.Info(errMessage)
@@ -937,9 +941,11 @@ func (m *MachineManager) chooseHost(ctx context.Context) (*bmov1alpha1.BareMetal
 		// If there are no hosts with nodeReuseLabelName, fall back
 		// to the current flow and select hosts randomly.
 		m.Log.Info("host(s) count available, choosing a random host", "availabeHostCount", len(availableHosts))
-		rHost, _ := rand.Int(rand.Reader, big.NewInt(int64(len(availableHosts))))
-		randomHost := rHost.Int64()
-		chosenHost = availableHosts[randomHost]
+		chosenHost, err = m.pickHost(availableHosts)
+		if err != nil {
+			m.Log.Error(err, "Failed to choose host, not choosing host")
+			return nil, nil, err
+		}
 	}
 
 	helper, err := v1beta1patch.NewHelper(chosenHost, m.client)
@@ -2115,3 +2121,46 @@ func (m *MachineManager) duplicateProviderIDsExist(validNodes map[string][]corev
 	}
 	return nil
 }
+
+// Picks host from list of available hosts, if failureDomain is set, tries to choose from hosts in failureDomain.
+// When none available in failureDomain it chooses from all available hosts.
+func (m *MachineManager) pickHost(availableHosts []*bmov1alpha1.BareMetalHost) (*bmov1alpha1.BareMetalHost, error) {
+	var chosenHost *bmov1alpha1.BareMetalHost
+	var availableHostsInFailureDomain []*bmov1alpha1.BareMetalHost
+
+	// When failureDomain is set, create a list from available hosts in failureDomain
+	if m.Metal3Machine.Spec.FailureDomain != "" {
+		labelSelector := labels.NewSelector()
+		var reqs labels.Requirements
+		var r *labels.Requirement
+		r, err := labels.NewRequirement(FailureDomainLabelName, selection.Equals, []string{m.Metal3Machine.Spec.FailureDomain})
+
+		if err != nil {
+			m.Log.Error(err, "Failed to create FailureDomain MatchLabel requirement, not choosing host")
+			return nil, err
+		}
+		reqs = append(reqs, *r)
+		labelSelector = labelSelector.Add(reqs...)
+
+		for _, host := range availableHosts {
+			if labelSelector.Matches(labels.Set(host.ObjectMeta.Labels)) {
+				availableHostsInFailureDomain = append(availableHostsInFailureDomain, host)
+			}
+		}
+		if len(availableHostsInFailureDomain) == 0 {
+			m.Log.Info("No available hosts in FailureDomain", m.Metal3Machine.Spec.FailureDomain, "choosing from other available hosts")
+		}
+	}
+
+	if len(availableHostsInFailureDomain) > 0 {
+		rHost, _ := rand.Int(rand.Reader, big.NewInt(int64(len(availableHostsInFailureDomain))))
+		randomHost := rHost.Int64()
+		chosenHost = availableHostsInFailureDomain[randomHost]
+	} else {
+		rHost, _ := rand.Int(rand.Reader, big.NewInt(int64(len(availableHosts))))
+		randomHost := rHost.Int64()
+		chosenHost = availableHosts[randomHost]
+	}
+
+	return chosenHost, nil
+}

baremetal/metal3machine_manager.go

@@ -570,6 +570,18 @@ var _ = Describe("Metal3Machine manager", func() {
 				},
 			},
 		}
+		hostWithFailureDomainLabel := bmov1alpha1.BareMetalHost{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "hostWithFailureDomainLabel",
+				Namespace: namespaceName,
+				Labels:    map[string]string{FailureDomainLabelName: "my-fd-1"},
+			},
+			Status: bmov1alpha1.BareMetalHostStatus{
+				Provisioning: bmov1alpha1.ProvisionStatus{
+					State: bmov1alpha1.StateAvailable,
+				},
+			},
+		}
 		hostWithUnhealthyAnnotation := bmov1alpha1.BareMetalHost{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:        "hostWithUnhealthyAnnotation",
@@ -607,6 +619,19 @@ var _ = Describe("Metal3Machine manager", func() {
 				},
 			},
 		}
+		hostWithNodeReuseLabelSetToCPinFailureDomain := bmov1alpha1.BareMetalHost{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "hostWithNodeReuseLabelSetToCPinFailureDomain",
+				Namespace: namespaceName,
+				Labels:    map[string]string{nodeReuseLabelName: "cp-test1", FailureDomainLabelName: "my-fd-1"},
+			},
+			Spec: bmov1alpha1.BareMetalHostSpec{},
+			Status: bmov1alpha1.BareMetalHostStatus{
+				Provisioning: bmov1alpha1.ProvisionStatus{
+					State: bmov1alpha1.StateAvailable,
+				},
+			},
+		}
 		hostWithNodeReuseLabelStateNone := bmov1alpha1.BareMetalHost{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "hostWithNodeReuseLabelStateNone",
@@ -622,13 +647,13 @@ var _ = Describe("Metal3Machine manager", func() {
 		}
 
 		m3mconfig, infrastructureRef := newConfig("", map[string]string{},
-			[]infrav1.HostSelectorRequirement{},
+			[]infrav1.HostSelectorRequirement{}, "",
 		)
 		m3mconfig2, infrastructureRef2 := newConfig("",
-			map[string]string{"key1": "value1"}, []infrav1.HostSelectorRequirement{},
+			map[string]string{"key1": "value1"}, []infrav1.HostSelectorRequirement{}, "",
 		)
 		m3mconfig3, infrastructureRef3 := newConfig("",
-			map[string]string{"boguskey": "value"}, []infrav1.HostSelectorRequirement{},
+			map[string]string{"boguskey": "value"}, []infrav1.HostSelectorRequirement{}, "",
 		)
 		m3mconfig4, infrastructureRef4 := newConfig("", map[string]string{},
 			[]infrav1.HostSelectorRequirement{
@@ -638,6 +663,7 @@ var _ = Describe("Metal3Machine manager", func() {
 					Values:   []string{"abc", "value1", "123"},
 				},
 			},
+			"",
 		)
 		m3mconfig5, infrastructureRef5 := newConfig("", map[string]string{},
 			[]infrav1.HostSelectorRequirement{
@@ -647,6 +673,10 @@ var _ = Describe("Metal3Machine manager", func() {
 					Values:   []string{"abc", "value1", "123"},
 				},
 			},
+			"",
+		)
+		m3mconfig6, infrastructureRef6 := newConfig("", map[string]string{},
+			[]infrav1.HostSelectorRequirement{}, "my-fd-1",
 		)
 
 		type testCaseChooseHost struct {
@@ -849,6 +879,68 @@ var _ = Describe("Metal3Machine manager", func() {
 				M3Machine:        m3mconfig5,
 				ExpectedHostName: "",
 			}),
+			Entry("Choose a host in Failure Domain", testCaseChooseHost{
+				Machine:          newMachine(machineName, infrastructureRef6),
+				Hosts:            &bmov1alpha1.BareMetalHostList{Items: []bmov1alpha1.BareMetalHost{*availableHost, hostWithLabel, hostWithOtherConsRef, hostWithFailureDomainLabel}},
+				M3Machine:        m3mconfig6,
+				ExpectedHostName: hostWithFailureDomainLabel.Name,
+			}),
+			Entry("Choose available host, when hosts in FailureDomain not available", testCaseChooseHost{
+				Machine:          newMachine(machineName, infrastructureRef6),
+				Hosts:            &bmov1alpha1.BareMetalHostList{Items: []bmov1alpha1.BareMetalHost{*availableHost, hostWithOtherConsRef, hostWithNodeReuseLabelSetToCP}},
+				M3Machine:        m3mconfig6,
+				ExpectedHostName: availableHost.Name,
+			}),
+			Entry("Choose a host in Failure Domain, when NodeReuse is set", testCaseChooseHost{
+				Machine: &clusterv1.Machine{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      machineName,
+						Namespace: namespaceName,
+
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								APIVersion: "controlplane.cluster.x-k8s.io/v1beta2",
+								Name:       "test1",
+								Kind:       "KubeadmControlPlane",
+							},
+						},
+						Labels: map[string]string{
+							clusterv1.MachineControlPlaneLabel: "cluster.x-k8s.io/control-plane",
+						},
+					},
+					Spec: clusterv1.MachineSpec{
+						InfrastructureRef: *infrastructureRef,
+					},
+				},
+				Hosts:            &bmov1alpha1.BareMetalHostList{Items: []bmov1alpha1.BareMetalHost{*availableHost, hostWithLabel, hostWithFailureDomainLabel, hostWithNodeReuseLabelSetToCPinFailureDomain}},
+				M3Machine:        m3mconfig6,
+				ExpectedHostName: hostWithNodeReuseLabelSetToCPinFailureDomain.Name,
+			}),
+			Entry("Choose host is not in Failure Domain, when NodeReuse is set", testCaseChooseHost{
+				Machine: &clusterv1.Machine{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      machineName,
+						Namespace: namespaceName,
+
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								APIVersion: "controlplane.cluster.x-k8s.io/v1beta2",
+								Name:       "test1",
+								Kind:       "KubeadmControlPlane",
+							},
+						},
+						Labels: map[string]string{
+							clusterv1.MachineControlPlaneLabel: "cluster.x-k8s.io/control-plane",
+						},
+					},
+					Spec: clusterv1.MachineSpec{
+						InfrastructureRef: *infrastructureRef,
+					},
+				},
+				Hosts:            &bmov1alpha1.BareMetalHostList{Items: []bmov1alpha1.BareMetalHost{*availableHost, hostWithLabel, hostWithFailureDomainLabel, hostWithNodeReuseLabelSetToCP}},
+				M3Machine:        m3mconfig6,
+				ExpectedHostName: hostWithNodeReuseLabelSetToCP.Name,
+			}),
 		)
 	})
 
@@ -1085,7 +1177,7 @@ var _ = Describe("Metal3Machine manager", func() {
 			fakeClient := fake.NewClientBuilder().WithScheme(setupSchemeMm()).WithObjects(tc.Host).Build()
 
 			m3mconfig, infrastructureRef := newConfig(tc.UserDataNamespace,
-				map[string]string{}, []infrav1.HostSelectorRequirement{},
+				map[string]string{}, []infrav1.HostSelectorRequirement{}, "",
 			)
 			if tc.UseCustomDeploy != nil {
 				m3mconfig.Spec.Image = infrav1.Image{}
@@ -1205,7 +1297,7 @@ var _ = Describe("Metal3Machine manager", func() {
 			fakeClient := fake.NewClientBuilder().WithScheme(setupSchemeMm()).WithObjects(tc.Host).Build()
 
 			m3mconfig, infrastructureRef := newConfig(tc.UserDataNamespace,
-				map[string]string{}, []infrav1.HostSelectorRequirement{},
+				map[string]string{}, []infrav1.HostSelectorRequirement{}, "",
 			)
 			machine := newMachine(machineName, infrastructureRef)
 
@@ -4579,7 +4671,7 @@ func setupSchemeMm() *runtime.Scheme {
 }
 
 func newConfig(userDataNamespace string,
-	labels map[string]string, reqs []infrav1.HostSelectorRequirement,
+	labels map[string]string, reqs []infrav1.HostSelectorRequirement, failureDomain string,
 ) (*infrav1.Metal3Machine, *clusterv1.ContractVersionedObjectReference) {
 	config := infrav1.Metal3Machine{
 		ObjectMeta: metav1.ObjectMeta{
@@ -4599,6 +4691,7 @@ func newConfig(userDataNamespace string,
 				MatchLabels:      labels,
 				MatchExpressions: reqs,
 			},
+			FailureDomain: failureDomain,
 		},
 		Status: infrav1.Metal3MachineStatus{
 			UserData: &corev1.SecretReference{