[backport] [2.11] add validation for etcd s3 cloud credential (#1021)

Author: jiaqiluo

docs.md

@@ -545,6 +545,10 @@ following:
 - Equal to another data directory
 - Attempts to nest another data directory
 
+##### Etcd S3 CloudCredential Secret
+
+Prevent the creation of objects if the secret specified in `.spec.rkeConfig.etcd.s3.cloudCredentialName` does not exist.
+
 #### On Update
 
 ##### Creator ID Annotation
@@ -561,7 +565,7 @@ section. A secondary validator will ensure that the effective data directory for
 from the one chosen during cluster creation. Additionally, the changing of a data directory for the `system-agent`, 
 kubernetes distro (RKE2/K3s), and CAPR components is also prohibited.
 
-#### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
+##### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
 
 The `DeploymentCustomization` fields are of 3 types:
 - `appendTolerations`: adds tolerations to the appropriate deployment (cluster-agent/fleet-agent)
@@ -576,7 +580,7 @@ A `Toleration` is matched to a regex which is provided by upstream [apimachinery
 
 For the `Affinity` based rules, the `podAffinity`/`podAntiAffinity` are validated via label selectors via [this apimachinery function](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L56) whereas the `nodeAffinity` `nodeSelectorTerms` are validated via the same `Toleration` function.
 
-#### cluster.spec.clusterAgentDeploymentCustomization.schedulingCustomization
+##### cluster.spec.clusterAgentDeploymentCustomization.schedulingCustomization
 
 The `SchedulingCustomization` subfield of the `DeploymentCustomization` field defines the properties of a Pod Disruption Budget and Priority Class which will be automatically deployed by Rancher for the cattle-cluster-agent.
 
@@ -595,10 +599,16 @@ Both `minAvailable` and `maxUnavailable` must be a string which represents a non
 ^([0-9]|[1-9][0-9]|100)%$
 ```
 
+##### Etcd S3 CloudCredential Secret
+
+Prevent the update of objects if the secret specified in `.spec.rkeConfig.etcd.s3.cloudCredentialName` does not exist.
+
 ### Mutation Checks
 
 #### On Create
 
+##### Creator ID Annotation
+
 When a cluster is created `field.cattle.io/creatorId` is set to the Username from the request.
 
 If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creatorId` does not get set.

pkg/resources/provisioning.cattle.io/v1/cluster/Cluster.md

@@ -19,6 +19,10 @@ following:
 - Equal to another data directory
 - Attempts to nest another data directory
 
+#### Etcd S3 CloudCredential Secret
+
+Prevent the creation of objects if the secret specified in `.spec.rkeConfig.etcd.s3.cloudCredentialName` does not exist.
+
 ### On Update
 
 #### Creator ID Annotation
@@ -35,7 +39,7 @@ section. A secondary validator will ensure that the effective data directory for
 from the one chosen during cluster creation. Additionally, the changing of a data directory for the `system-agent`, 
 kubernetes distro (RKE2/K3s), and CAPR components is also prohibited.
 
-### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
+#### cluster.spec.clusterAgentDeploymentCustomization and cluster.spec.fleetAgentDeploymentCustomization
 
 The `DeploymentCustomization` fields are of 3 types:
 - `appendTolerations`: adds tolerations to the appropriate deployment (cluster-agent/fleet-agent)
@@ -50,7 +54,7 @@ A `Toleration` is matched to a regex which is provided by upstream [apimachinery
 
 For the `Affinity` based rules, the `podAffinity`/`podAntiAffinity` are validated via label selectors via [this apimachinery function](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L56) whereas the `nodeAffinity` `nodeSelectorTerms` are validated via the same `Toleration` function.
 
-### cluster.spec.clusterAgentDeploymentCustomization.schedulingCustomization
+#### cluster.spec.clusterAgentDeploymentCustomization.schedulingCustomization
 
 The `SchedulingCustomization` subfield of the `DeploymentCustomization` field defines the properties of a Pod Disruption Budget and Priority Class which will be automatically deployed by Rancher for the cattle-cluster-agent.
 
@@ -69,10 +73,16 @@ Both `minAvailable` and `maxUnavailable` must be a string which represents a non
 ^([0-9]|[1-9][0-9]|100)%$
 ```
 
+#### Etcd S3 CloudCredential Secret
+
+Prevent the update of objects if the secret specified in `.spec.rkeConfig.etcd.s3.cloudCredentialName` does not exist.
+
 ## Mutation Checks
 
 ### On Create
 
+#### Creator ID Annotation
+
 When a cluster is created `field.cattle.io/creatorId` is set to the Username from the request.
 
 If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creatorId` does not get set.

pkg/resources/provisioning.cattle.io/v1/cluster/validator.go

@@ -53,6 +53,7 @@ func NewProvisioningClusterValidator(client *clients.Clients) *ProvisioningClust
 		admitter: provisioningAdmitter{
 			sar:               client.K8s.AuthorizationV1().SubjectAccessReviews(),
 			mgmtClusterClient: client.Management.Cluster(),
+			secretClient:      client.Core.Secret(),
 			secretCache:       client.Core.Secret().Cache(),
 			psactCache:        client.Management.PodSecurityAdmissionConfigurationTemplate().Cache(),
 			featureCache:      client.Management.Feature().Cache(),
@@ -87,6 +88,7 @@ func (p *ProvisioningClusterValidator) Admitters() []admission.Admitter {
 type provisioningAdmitter struct {
 	sar               authorizationv1.SubjectAccessReviewInterface
 	mgmtClusterClient v3.ClusterClient
+	secretClient      corev1controller.SecretController
 	secretCache       corev1controller.SecretCache
 	psactCache        v3.PodSecurityAdmissionConfigurationTemplateCache
 	featureCache      v3.FeatureCache
@@ -150,6 +152,10 @@ func (p *provisioningAdmitter) Admit(request *admission.Request) (*admissionv1.A
 		if response, err = p.validatePriorityClass(oldCluster, cluster); err != nil || !response.Allowed {
 			return response, err
 		}
+
+		if response, err = p.validateS3Secret(oldCluster, cluster); err != nil || !response.Allowed {
+			return response, err
+		}
 	}
 
 	if err := p.validatePSACT(request, response, cluster); err != nil || response.Result != nil {
@@ -670,6 +676,51 @@ func (p *provisioningAdmitter) validatePodDisruptionBudget(oldCluster, cluster *
 	return admission.ResponseAllowed(), nil
 }
 
+// validateS3Secret checks if the S3 cloud credential secret referenced in the cluster exists.
+func (p *provisioningAdmitter) validateS3Secret(oldCluster, cluster *v1.Cluster) (*admissionv1.AdmissionResponse, error) {
+	if cluster.Name == localCluster {
+		return admission.ResponseAllowed(), nil
+	}
+
+	rkeConfig := cluster.Spec.RKEConfig
+	if rkeConfig == nil || rkeConfig.ETCD == nil || rkeConfig.ETCD.S3 == nil || rkeConfig.ETCD.S3.CloudCredentialName == "" {
+		return admission.ResponseAllowed(), nil
+	}
+
+	var (
+		oldSecret string
+		newSecret string
+	)
+	if oldCluster.Spec.RKEConfig != nil &&
+		oldCluster.Spec.RKEConfig.ETCD != nil &&
+		oldCluster.Spec.RKEConfig.ETCD.S3 != nil {
+		oldSecret = oldCluster.Spec.RKEConfig.ETCD.S3.CloudCredentialName
+	}
+	newSecret = cluster.Spec.RKEConfig.ETCD.S3.CloudCredentialName
+
+	// Skip if the credential names remain the same
+	if oldSecret == newSecret {
+		return admission.ResponseAllowed(), nil
+	}
+
+	// check if the secret exists
+	namespace, name := getCloudCredentialSecretInfo(cluster.Namespace, newSecret)
+	_, err := p.secretCache.Get(namespace, name)
+	if apierrors.IsNotFound(err) {
+		// Since the secret can be created alongside the cluster via the UI, there's a chance it's not yet present in the cache.
+		// Therefore, we perform an additional check directly against the API server.
+		_, err = p.secretClient.Get(namespace, name, metav1.GetOptions{})
+	}
+	if apierrors.IsNotFound(err) {
+		msg := fmt.Sprintf("etcd s3 cloud credential secret %s/%s is not found", namespace, name)
+		return admission.ResponseBadRequest(msg), nil
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to get etcd s3 cloud credential secret %s/%s: %w", namespace, name, err)
+	}
+
+	return admission.ResponseAllowed(), nil
+}
+
 func validateAgentDeploymentCustomization(customization *v1.AgentDeploymentCustomization, path *field.Path) field.ErrorList {
 	if customization == nil {
 		return nil