CloudWatch infra support for eksapi deployer (#648)

* CW Infra changes and supporting files

* cw infra changes as suggested

* Pod Agent Identity

* fixing for nodeam update

* Fixing nits

* infra.go uuid

* moree edits

* fixing nit.

Author: annapoornanarayan

internal/deployers/eksapi/deployer.go

@@ -8,6 +8,8 @@ import (
 
 	"github.com/aws/aws-k8s-tester/internal"
 	"github.com/aws/aws-k8s-tester/internal/awssdk"
+	"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
+	fwext "github.com/aws/aws-k8s-tester/internal/e2e"
 	"github.com/aws/aws-k8s-tester/internal/metrics"
 	"github.com/aws/aws-k8s-tester/internal/util"
 
@@ -232,6 +234,22 @@ func (d *deployer) Up() error {
 			// don't return err, this isn't critical
 		}
 	}
+
+	klog.Infof("Setting up CloudWatch infrastructure...")
+	if roleArn, err := d.infraManager.createCloudWatchInfrastructureStack(d.cluster.name); err != nil {
+		klog.Errorf("CloudWatch infrastructure setup failed: %v", err)
+		return err
+	} else {
+		d.infra.cloudwatchRoleArn = roleArn
+		klog.Infof("CloudWatch infrastructure setup completed")
+	}
+	// Apply CloudWatch infrastructure manifest
+	manifest := templates.CloudWatchAgentRbac
+	if err := fwext.ApplyManifests(d.k8sClient.config, manifest); err != nil {
+		klog.Errorf("CloudWatch infrastructure manifest failed: %v", err)
+		return err
+	}
+	klog.Infof("CloudWatch infrastructure manifest applied successfully")
 	return nil
 }
 
@@ -338,6 +356,9 @@ func (d *deployer) Down() error {
 }
 
 func deleteResources(im *InfrastructureManager, cm *ClusterManager, nm *nodeManager, k8sClient *k8sClient /* nillable */, opts *deployerOptions /* nillable */) error {
+	if err := im.deleteCloudWatchInfrastructureStack(); err != nil {
+		return err
+	}
 	if err := nm.deleteNodes(k8sClient, opts); err != nil {
 		return err
 	}

internal/deployers/eksapi/infra.go

@@ -85,6 +85,7 @@ type Infrastructure struct {
 	clusterRoleARN    string
 	nodeRoleARN       string
 	nodeRoleName      string
+	cloudwatchRoleArn string
 }
 
 func (i *Infrastructure) subnets() []string {
@@ -475,3 +476,77 @@ func (m *InfrastructureManager) getAZsWithCapacity(opts *deployerOptions) ([]str
 	}
 	return subnetAzs, nil
 }
+
+func getCloudWatchStackName(resourceID string) (string, string) {
+	clusterUUID := strings.TrimPrefix(resourceID, ResourcePrefix+"-")
+	return fmt.Sprintf("cloudwatch-%s", clusterUUID), clusterUUID
+}
+
+func (m *InfrastructureManager) createCloudWatchInfrastructureStack(clusterName string) (string, error) {
+	stackName, clusterUUID := getCloudWatchStackName(clusterName)
+	klog.Infof("creating CloudWatch infrastructure stack: %s", stackName)
+	out, err := m.clients.CFN().CreateStack(context.TODO(), &cloudformation.CreateStackInput{
+		StackName:    aws.String(stackName),
+		TemplateBody: aws.String(templates.CloudWatchInfra),
+		Capabilities: []cloudformationtypes.Capability{cloudformationtypes.CapabilityCapabilityNamedIam},
+		Parameters: []cloudformationtypes.Parameter{
+			{
+				ParameterKey:   aws.String("ClusterName"),
+				ParameterValue: aws.String(clusterName),
+			},
+			{
+				ParameterKey:   aws.String("ClusterUUID"),
+				ParameterValue: aws.String(clusterUUID),
+			},
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to create CloudWatch infrastructure stack: %w", err)
+	}
+
+	klog.Infof("waiting for CloudWatch infrastructure stack to be created: %s", *out.StackId)
+	if err := cloudformation.NewStackCreateCompleteWaiter(m.clients.CFN()).
+		Wait(context.TODO(),
+			&cloudformation.DescribeStacksInput{
+				StackName: out.StackId,
+			},
+			infraStackCreationTimeout); err != nil {
+		return "", fmt.Errorf("failed to wait for CloudWatch infrastructure stack creation: %w", err)
+	}
+
+	// Get the CloudWatch role ARN from stack outputs
+	stack, err := m.clients.CFN().DescribeStacks(context.TODO(), &cloudformation.DescribeStacksInput{
+		StackName: out.StackId,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to describe CloudWatch infrastructure stack: %w", err)
+	}
+
+	for _, output := range stack.Stacks[0].Outputs {
+		if aws.ToString(output.OutputKey) == "CloudWatchRoleArn" {
+			klog.Infof("CloudWatch infrastructure stack created successfully with role ARN: %s", aws.ToString(output.OutputValue))
+			return aws.ToString(output.OutputValue), nil
+		}
+	}
+
+	return "", fmt.Errorf("CloudWatch role ARN not found in stack outputs")
+}
+
+func (m *InfrastructureManager) deleteCloudWatchInfrastructureStack() error {
+	stackName, _ := getCloudWatchStackName(m.resourceID)
+
+	klog.Infof("deleting CloudWatch infrastructure stack: %s", stackName)
+	if _, err := m.clients.CFN().DeleteStack(context.TODO(), &cloudformation.DeleteStackInput{
+		StackName: aws.String(stackName),
+	}); err != nil {
+		var notFound *cloudformationtypes.StackNotFoundException
+		if errors.As(err, &notFound) {
+			klog.Infof("CloudWatch infrastructure stack does not exist: %s", stackName)
+			return nil
+		}
+		return fmt.Errorf("failed to delete CloudWatch infrastructure stack: %w", err)
+	}
+
+	klog.Infof("initiated deletion of CloudWatch infrastructure stack: %s", stackName)
+	return nil
+}

internal/deployers/eksapi/templates/cloudwatch-infra.yaml.template

@@ -0,0 +1,57 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: kubetest2-eksapi CloudWatch using Pod Identity
+
+Parameters:
+  ClusterName:
+    Description: Name of the EKS cluster
+    Type: String
+  
+  ClusterUUID:
+    Description: UUID portion of the cluster name
+    Type: String
+
+Resources:
+  CloudWatchRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub "cloudwatch-role-${ClusterUUID}"
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Sid: AllowEksAuthToAssumeRoleForPodIdentity
+            Effect: Allow
+            Principal:
+              Service: pods.eks.amazonaws.com
+            Action:
+              - sts:AssumeRole
+              - sts:TagSession
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy
+      Description: Role for CloudWatch Agent in EKS cluster
+
+  PodIdentityAssociation:
+    Type: AWS::EKS::PodIdentityAssociation
+    Properties:
+      ClusterName: !Ref ClusterName
+      Namespace: amazon-cloudwatch
+      ServiceAccount: cwagent
+      RoleArn: !GetAtt CloudWatchRole.Arn
+  
+  EksPodIdentityAgentAddon:
+    Type: AWS::EKS::Addon
+    Properties:
+      AddonName: eks-pod-identity-agent
+      ClusterName: !Ref ClusterName
+
+Outputs:
+  CloudWatchRoleArn:
+    Description: ARN of the CloudWatch IAM role
+    Value: !GetAtt CloudWatchRole.Arn
+    Export:
+      Name: !Sub "${AWS::StackName}::CloudWatchRoleArn"
+
+  PodIdentityAssociationArn:
+    Description: ARN of the Pod Identity Association
+    Value: !Ref PodIdentityAssociation
+    Export:
+      Name: !Sub '${AWS::StackName}-PodIdentityAssociationArn'

internal/deployers/eksapi/templates/cloudwatch_agent_infra.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: amazon-cloudwatch
+  labels:
+    name: amazon-cloudwatch
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cwagent
+  namespace: amazon-cloudwatch
+
+---
+# ClusterRole for cwagent
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cwagent-role
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["extensions"]
+    resources:
+      - ingresses
+    verbs: ["get", "list", "watch"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+
+---
+# ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cwagent-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: cwagent
+    namespace: amazon-cloudwatch
+roleRef:
+  kind: ClusterRole
+  name: cwagent-role
+  apiGroup: rbac.authorization.k8s.io

internal/deployers/eksapi/templates/templates.go

@@ -8,12 +8,18 @@ import (
 //go:embed infra.yaml
 var Infrastructure string
 
+//go:embed cloudwatch_agent_infra.yaml
+var CloudWatchAgentRbac []byte
+
 var (
 	//go:embed unmanaged-nodegroup.yaml.template
 	unmanagedNodegroupTemplate string
 	UnmanagedNodegroup         = template.Must(template.New("unmanagedNodegroup").Parse(unmanagedNodegroupTemplate))
 )
 
+//go:embed cloudwatch-infra.yaml.template
+var CloudWatchInfra string
+
 type NetworkInterface struct {
 	Description         *string
 	NetworkCardIndex    *int