Add ALPHA metadata-labeler sidecar and metadata source

Co-authored-by: Drew Sirenko <[email protected]>
Co-authored-by: Connor Catlett <[email protected]>
Signed-off-by: Connor Catlett <[email protected]>

Author: 3 people

Makefile

@@ -147,9 +147,11 @@ e2e/multi-az: bin/helm bin/ginkgo
 .PHONY: e2e/disruptive
 e2e/disruptive: bin/helm bin/ginkgo
 	TEST_PATH=./tests/e2e/... \
-	GINKGO_FOCUS="\[ebs-csi-e2e\] \[disruptive\]" \
+	GINKGO_FOCUS="\[ebs-csi-e2e\] \[Disruptive\]" \
+	GINKGO_SKIP="\[Flaky\]" \
 	GINKGO_PARALLEL=1 \
 	EBS_INSTALL_SNAPSHOT=false \
+	HELM_EXTRA_FLAGS="--set=sidecars.metadataLabeler.enabled=true,node.metadataSources='metadata-labeler'" \
 	./hack/e2e/run.sh
 
 .PHONY: e2e/external

charts/aws-ebs-csi-driver/templates/clusterrole-metadata-labeler.yaml

@@ -0,0 +1,16 @@
+{{- if and (not .Values.nodeComponentOnly) (.Values.sidecars.metadataLabeler.enabled) -}}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-metadata-labeler-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["patch"]
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["list", "watch", "get"]
+{{- end -}}

charts/aws-ebs-csi-driver/templates/clusterrolebinding-metadata-labeler.yaml

@@ -0,0 +1,17 @@
+{{- if and (not .Values.nodeComponentOnly) (.Values.sidecars.metadataLabeler.enabled) -}}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-metadata-labeler-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.controller.serviceAccount.name }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-metadata-labeler-role
+{{- end -}}

charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -489,6 +489,40 @@ spec:
           {{- end }}
           terminationMessagePolicy: FallbackToLogsOnError
         {{- end }}
+        {{- if .Values.sidecars.metadataLabeler.enabled }}
+        - name: metadata-labeler
+          image: {{ include "aws-ebs-csi-driver.fullImagePath" $ }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - metadataLabeler
+            - --v={{ .Values.sidecars.metadataLabeler.logLevel }}
+            {{- range .Values.sidecars.metadataLabeler.additionalArgs }}
+            - {{ . }}
+            {{- end }}
+          env:
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.metadataLabeler.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          {{- with .Values.controller.envFrom }}
+          envFrom:
+            {{- . | toYaml | nindent 12 }}
+          {{- end }}
+          {{- with default .Values.controller.resources .Values.sidecars.metadataLabeler.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.metadataLabeler.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
         - name: csi-resizer
           image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.resizer.image.repository .Values.sidecars.resizer.image.tag }}
           imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.resizer.image.pullPolicy }}

charts/aws-ebs-csi-driver/templates/csidriver.yaml

@@ -9,8 +9,10 @@ spec:
   attachRequired: true
   podInfoOnMount: false
   {{- if semverCompare ">=1.33.0-0" .Capabilities.KubeVersion.Version }}
-  {{- with .Values.nodeAllocatableUpdatePeriodSeconds }}
-  nodeAllocatableUpdatePeriodSeconds: {{ . }}
+  {{- if eq (.Values.nodeAllocatableUpdatePeriodSeconds | int) -1 }}
+  nodeAllocatableUpdatePeriodSeconds: {{ (regexMatch "metadata-labeler" (.Values.node.metadataSources | default "")) | ternary "300" "10" }}
+  {{- else if .Values.nodeAllocatableUpdatePeriodSeconds }}
+  nodeAllocatableUpdatePeriodSeconds: {{ .Values.nodeAllocatableUpdatePeriodSeconds }}
   {{- end }}
   {{- end }}
   {{- if not .Values.useOldCSIDriver }}

charts/aws-ebs-csi-driver/values.schema.json

@@ -108,8 +108,8 @@
     },
     "nodeAllocatableUpdatePeriodSeconds": {
       "type": ["integer", "null"],
-      "description": "nodeAllocatableUpdatePeriodSeconds updates the node's max attachable volume count by directing Kubelet to periodically call NodeGetInfo at the configured interval. Kubernetes enforces a minimum update interval of 10 seconds. This parameter is supported in Kubernetes 1.33+, the MutableCSINodeAllocatableCount feature gate must be enabled in kubelet and kube-apiserver.",
-      "default": 10
+      "description": "nodeAllocatableUpdatePeriodSeconds updates the node's max attachable volume count by directing Kubelet to periodically call NodeGetInfo at the configured interval. Kubernetes enforces a minimum update interval of 10 seconds. A value of -1 uses a automatically determined value dependent on metadata sources. This parameter is supported in Kubernetes 1.33+, the MutableCSINodeAllocatableCount feature gate must be enabled in kubelet and kube-apiserver.",
+      "default": -1
     },
     "nodeComponentOnly": {
       "type": "boolean",
@@ -673,7 +673,7 @@
           "default": false
         },
         "metadataSources": {
-          "description": "Comma separated list of metadata sources that override the default used by the EBS CSI Driver. Valid sources include 'imds' and 'kubernetes'",
+          "description": "Comma separated list of metadata sources that override the default used by the EBS CSI Driver. Valid sources include 'imds', 'kubernetes', and (ALPHA) 'metadata-labeler'",
           "type": ["string", "null"],
           "default": null
         },
@@ -1163,6 +1163,41 @@
             }
           }
         },
+        "metadataLabeler": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "additionalArgs": {
+              "type": "array",
+              "description": "Additional arguments passed to the metadataLabeler container",
+              "default": [],
+              "items": {
+                "type": "string"
+              }
+            },
+            "resources": {
+              "type": ["object", "null"],
+              "default": null
+            },
+            "enabled": {
+              "type": "boolean",
+              "description": "ALPHA: Enable the metadata-labeler sidecar to label Kubernetes Nodes with information from the EC2 API (e.g. number of ENIs). Also requires using metadata-labeler as the node's metadata source.",
+              "default": false
+            },
+            "logLevel": {
+              "type": "integer",
+              "description": "Set the level of verbosity of the logs",
+              "default": 4
+            },
+            "env": {
+              "type": "array",
+              "default": []
+            },
+            "securityContext": {
+              "type": "object"
+            }
+          }
+        },
         "volumemodifier": {
           "type": "object",
           "additionalProperties": false,

charts/aws-ebs-csi-driver/values.yaml

@@ -90,6 +90,18 @@ sidecars:
         type: RuntimeDefault
       readOnlyRootFilesystem: true
       allowPrivilegeEscalation: false
+  metadataLabeler:
+    # ALPHA: Enable the metadata-labeler sidecar to label Kubernetes Nodes with
+    # information from the EC2 API (e.g. number of ENIs)
+    # Also requires using metadata-labeler as the node's metadata source
+    enabled: false
+    logLevel: 2
+    # Additional parameters provided by metadataLabeler.
+    additionalArgs: []
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
   livenessProbe:
     image:
       pullPolicy: IfNotPresent
@@ -425,6 +437,7 @@ node:
   # Enable the linux daemonset creation
   enableLinux: true
   enableWindows: true
+  # Comma separated list of metadata sources that override the default used by the EBS CSI Driver. Valid sources include 'imds', 'kubernetes', and (ALPHA) 'metadata-labeler'
   metadataSources:
   # Warning: This option will be removed in a future release. It is a temporary workaround for users unable to immediately migrate off of older kernel versions.
   # Formats XFS volumes with bigtime=0,inobtcount=0,reflink=0, for mounting onto nodes with linux kernel version <= 5.4.
@@ -527,9 +540,9 @@ volumeSnapshotClasses: []
 # This parameter should always be false for new installations
 useOldCSIDriver: false
 # nodeAllocatableUpdatePeriodSeconds updates the node's max attachable volume count by directing Kubelet to periodically call NodeGetInfo at the configured interval.
-# Kubernetes enforces a minimum update interval of 10 seconds.
+# Kubernetes enforces a minimum update interval of 10 seconds. A value of -1 uses a automatically determined value dependent on metadata sources.
 # This parameter is supported in Kubernetes 1.33+ and requires the MutableCSINodeAllocatableCount feature gate to be enabled in kubelet and kube-apiserver.
-nodeAllocatableUpdatePeriodSeconds: 10
+nodeAllocatableUpdatePeriodSeconds: -1
 # Deploy EBS CSI Driver without controller and associated resources
 nodeComponentOnly: false
 # Set maximum verbosity for logs of each container and other recommended debugging parameters such as enabling AWS SDK debug logging

cmd/main.go

@@ -24,12 +24,13 @@ import (
 	"time"
 
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/cmd/hooks"
-	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
+	cloudPkg "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud/metadata"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/metrics"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter"
 	flag "github.com/spf13/pflag"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/featuregate"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	json "k8s.io/component-base/logs/json"
@@ -54,6 +55,15 @@ func main() {
 		options  = driver.Options{}
 	)
 
+	var (
+		metadataRequiredModes = map[string]struct{}{
+			string(driver.ControllerMode): {},
+			string(driver.NodeMode):       {},
+			string(driver.AllMode):        {},
+			driver.MetadataLabelerMode:    {},
+		}
+	)
+
 	c := logsapi.NewLoggingConfiguration()
 	err := logsapi.AddFeatureGates(featureGate)
 	if err != nil {
@@ -66,6 +76,65 @@ func main() {
 		args = os.Args[2:]
 	}
 
+	options.Mode = driver.Mode(cmd)
+	options.AddFlags(fs)
+
+	if err := fs.Parse(args); err != nil {
+		klog.ErrorS(err, "Failed to parse options")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
+	}
+	if err := options.Validate(); err != nil {
+		klog.ErrorS(err, "Invalid options")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
+	}
+
+	err = logsapi.ValidateAndApply(c, featureGate)
+	if err != nil {
+		klog.ErrorS(err, "failed to validate and apply logging configuration")
+	}
+
+	var cloud cloudPkg.Cloud
+	var k8sClient kubernetes.Interface
+	var md metadata.MetadataService
+	cfg := metadata.MetadataServiceConfig{
+		MetadataSources: options.MetadataSources,
+		IMDSClient:      metadata.DefaultIMDSClient,
+		K8sAPIClient:    metadata.DefaultKubernetesAPIClient(options.Kubeconfig),
+	}
+
+	if _, ok := metadataRequiredModes[cmd]; ok {
+		region := os.Getenv("AWS_REGION")
+		var metadataErr error
+
+		if region != "" {
+			klog.InfoS("Region provided via AWS_REGION environment variable", "region", region)
+			if options.Mode != driver.ControllerMode {
+				klog.InfoS("Node service requires metadata even if AWS_REGION provided, initializing metadata")
+				md, metadataErr = metadata.NewMetadataService(cfg, region)
+			}
+		} else {
+			klog.InfoS("Initializing metadata")
+			md, metadataErr = metadata.NewMetadataService(cfg, region)
+		}
+
+		if metadataErr != nil {
+			klog.ErrorS(metadataErr, "Failed to initialize metadata when it is required")
+			if options.Mode == driver.ControllerMode {
+				klog.InfoS("The region can be manually supplied via the AWS_REGION environment variable")
+			}
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		} else if region == "" {
+			region = md.GetRegion()
+		}
+
+		cloud = cloudPkg.NewCloud(region, options.AwsSdkDebugLog, options.UserAgentExtra, options.Batching, options.DeprecatedMetrics)
+	}
+
+	k8sClient, err = cfg.K8sAPIClient()
+	if err != nil {
+		klog.V(2).InfoS("Failed to setup k8s client", "err", err)
+	}
+
 	switch cmd {
 	case "pre-stop-hook":
 		clientset, clientErr := metadata.DefaultKubernetesAPIClient(options.Kubeconfig)()
@@ -80,28 +149,17 @@ func main() {
 		}
 		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
 	case string(driver.ControllerMode), string(driver.NodeMode), string(driver.AllMode):
-		options.Mode = driver.Mode(cmd)
+	case driver.MetadataLabelerMode:
+		err := metadata.ContinuousUpdateLabelsLeaderElection(k8sClient, cloud, metadata.ControllerMetadataLabelerInterval)
+		if err != nil {
+			klog.ErrorS(err, "failed to patch volume/ENI count on node labels")
+			klog.FlushAndExit(klog.ExitFlushTimeout, 0)
+		}
 	default:
-		klog.Errorf("Unknown driver mode %s: Expected %s, %s, %s, or pre-stop-hook", cmd, driver.ControllerMode, driver.NodeMode, driver.AllMode)
+		klog.Errorf("Unknown driver mode %s: Expected %s, %s, %s, %s, or pre-stop-hook", cmd, driver.ControllerMode, driver.NodeMode, driver.AllMode, driver.MetadataLabelerMode)
 		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
 	}
 
-	options.AddFlags(fs)
-
-	if err = fs.Parse(args); err != nil {
-		klog.ErrorS(err, "Failed to parse options")
-		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
-	}
-	if err = options.Validate(); err != nil {
-		klog.ErrorS(err, "Invalid options")
-		klog.FlushAndExit(klog.ExitFlushTimeout, 0)
-	}
-
-	err = logsapi.ValidateAndApply(c, featureGate)
-	if err != nil {
-		klog.ErrorS(err, "failed to validate and apply logging configuration")
-	}
-
 	if *version {
 		versionInfo, versionErr := driver.GetVersionJSON()
 		if versionErr != nil {
@@ -134,27 +192,6 @@ func main() {
 		}()
 	}
 
-	cfg := metadata.MetadataServiceConfig{
-		MetadataSources: options.MetadataSources,
-		IMDSClient:      metadata.DefaultIMDSClient,
-		K8sAPIClient:    metadata.DefaultKubernetesAPIClient(options.Kubeconfig),
-	}
-
-	region := os.Getenv("AWS_REGION")
-	var md metadata.MetadataService
-	var metadataErr error
-
-	if region != "" {
-		klog.InfoS("Region provided via AWS_REGION environment variable", "region", region)
-		if options.Mode != driver.ControllerMode {
-			klog.InfoS("Node service requires metadata even if AWS_REGION provided, initializing metadata")
-			md, metadataErr = metadata.NewMetadataService(cfg, region)
-		}
-	} else {
-		klog.InfoS("Initializing metadata")
-		md, metadataErr = metadata.NewMetadataService(cfg, region)
-	}
-
 	if options.HTTPEndpoint != "" {
 		r := metrics.InitializeRecorder(options.DeprecatedMetrics)
 		r.InitializeMetricsHandler(options.HTTPEndpoint, "/metrics", options.MetricsCertFile, options.MetricsKeyFile)
@@ -169,29 +206,12 @@ func main() {
 		}
 	}
 
-	if metadataErr != nil {
-		klog.ErrorS(metadataErr, "Failed to initialize metadata when it is required")
-		if options.Mode == driver.ControllerMode {
-			klog.InfoS("The region can be manually supplied via the AWS_REGION environment variable")
-		}
-		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
-	} else if region == "" {
-		region = md.GetRegion()
-	}
-
-	cloud := cloud.NewCloud(region, options.AwsSdkDebugLog, options.UserAgentExtra, options.Batching, options.DeprecatedMetrics)
-
 	m, err := mounter.NewNodeMounter(options.WindowsHostProcess)
 	if err != nil {
 		klog.ErrorS(err, "failed to create node mounter")
 		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
 	}
 
-	k8sClient, err := cfg.K8sAPIClient()
-	if err != nil {
-		klog.V(2).InfoS("Failed to setup k8s client", "err", err)
-	}
-
 	drv, err := driver.NewDriver(cloud, &options, m, md, k8sClient)
 	if err != nil {
 		klog.ErrorS(err, "failed to create driver")