Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions pkg/apis/v1/ec2nodeclass.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

awssdk "github.com/aws/karpenter-provider-aws/pkg/aws"
)

// EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider.
Expand Down Expand Up @@ -501,6 +503,37 @@ func (in *EC2NodeClass) Hash() string {
})))
}

// HashForRegion returns a hash of the EC2NodeClass with region-specific filtering applied.
// This ensures that unsupported features in certain regions don't cause drift.
func (in *EC2NodeClass) HashForRegion(region string) string {
spec := in.Spec.DeepCopy()

// Filter out HTTPProtocolIPv6 for regions that don't support it
// This matches the filtering logic in pkg/providers/amifamily/resolver.go
if lo.Contains(awssdk.HTTPProtocolUnsupportedRegions, region) && spec.MetadataOptions != nil {
// Create a copy of MetadataOptions without HTTPProtocolIPv6
filteredMetadataOptions := &MetadataOptions{
HTTPEndpoint: spec.MetadataOptions.HTTPEndpoint,
HTTPPutResponseHopLimit: spec.MetadataOptions.HTTPPutResponseHopLimit,
HTTPTokens: spec.MetadataOptions.HTTPTokens,
// HTTPProtocolIPv6 is intentionally omitted for unsupported regions
}
spec.MetadataOptions = filteredMetadataOptions
}

return fmt.Sprint(lo.Must(hashstructure.Hash([]interface{}{
spec,
// AMIFamily should be hashed using the dynamically resolved value rather than the literal value of the field.
// This ensures that scenarios such as changing the field from nil to AL2023 with the alias "al2023@latest"
// doesn't trigger drift.
in.AMIFamily(),
}, hashstructure.FormatV2, &hashstructure.HashOptions{
SlicesAsSets: true,
IgnoreZeroValue: true,
ZeroNil: true,
})))
}

func (in *EC2NodeClass) LegacyInstanceProfileName(clusterName, region string) string {
return fmt.Sprintf("%s_%d", clusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, in.Name), hashstructure.FormatV2, nil)))
}
Expand Down
72 changes: 72 additions & 0 deletions pkg/apis/v1/ec2nodeclass_hash_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ limitations under the License.
package v1_test

import (
"fmt"

"github.com/imdario/mergo"
"github.com/samber/lo"
"k8s.io/apimachinery/pkg/api/resource"
Expand All @@ -24,6 +26,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
awssdk "github.com/aws/karpenter-provider-aws/pkg/aws"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand Down Expand Up @@ -205,4 +208,73 @@ var _ = Describe("Hash", func() {
}
Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash()))
})

Context("Region-specific filtering", func() {
It("should filter HTTPProtocolIPv6 for unsupported regions", func() {
nodeClass.Spec.MetadataOptions = &v1.MetadataOptions{
HTTPEndpoint: lo.ToPtr("enabled"),
HTTPProtocolIPv6: lo.ToPtr("enabled"),
HTTPPutResponseHopLimit: lo.ToPtr(int64(1)),
HTTPTokens: lo.ToPtr("required"),
}

// Hash for supported region should include HTTPProtocolIPv6
supportedRegionHash := nodeClass.HashForRegion("us-west-2")

// Hash for unsupported region should exclude HTTPProtocolIPv6
unsupportedRegionHash := nodeClass.HashForRegion("us-iso-east-1")

// The hashes should be different because HTTPProtocolIPv6 is filtered out
Expect(supportedRegionHash).ToNot(Equal(unsupportedRegionHash))
})

It("should have same hash for unsupported regions regardless of HTTPProtocolIPv6 value", func() {
nodeClass1 := nodeClass.DeepCopy()
nodeClass1.Spec.MetadataOptions = &v1.MetadataOptions{
HTTPEndpoint: lo.ToPtr("enabled"),
HTTPProtocolIPv6: lo.ToPtr("enabled"),
HTTPPutResponseHopLimit: lo.ToPtr(int64(1)),
HTTPTokens: lo.ToPtr("required"),
}

nodeClass2 := nodeClass.DeepCopy()
nodeClass2.Spec.MetadataOptions = &v1.MetadataOptions{
HTTPEndpoint: lo.ToPtr("enabled"),
HTTPProtocolIPv6: lo.ToPtr("disabled"),
HTTPPutResponseHopLimit: lo.ToPtr(int64(1)),
HTTPTokens: lo.ToPtr("required"),
}

// Both should have the same hash for unsupported regions since HTTPProtocolIPv6 is filtered out
hash1 := nodeClass1.HashForRegion("us-iso-east-1")
hash2 := nodeClass2.HashForRegion("us-iso-east-1")
Expect(hash1).To(Equal(hash2))
})

It("should not filter HTTPProtocolIPv6 when MetadataOptions is nil", func() {
nodeClass.Spec.MetadataOptions = nil

supportedRegionHash := nodeClass.HashForRegion("us-west-2")
unsupportedRegionHash := nodeClass.HashForRegion("us-iso-east-1")

// Should be the same since there's nothing to filter
Expect(supportedRegionHash).To(Equal(unsupportedRegionHash))
})

It("should match all unsupported regions", func() {
nodeClass.Spec.MetadataOptions = &v1.MetadataOptions{
HTTPEndpoint: lo.ToPtr("enabled"),
HTTPProtocolIPv6: lo.ToPtr("enabled"),
HTTPPutResponseHopLimit: lo.ToPtr(int64(1)),
HTTPTokens: lo.ToPtr("required"),
}

supportedRegionHash := nodeClass.HashForRegion("us-west-2")

for _, region := range awssdk.HTTPProtocolUnsupportedRegions {
unsupportedRegionHash := nodeClass.HashForRegion(region)
Expect(unsupportedRegionHash).ToNot(Equal(supportedRegionHash), fmt.Sprintf("Region %s should filter HTTPProtocolIPv6", region))
}
})
})
})
10 changes: 10 additions & 0 deletions pkg/aws/sdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,16 @@ import (
"github.com/aws/aws-sdk-go-v2/service/timestreamwrite"
)

// HTTPProtocolUnsupportedRegions contains regions that don't support HTTPProtocolIPv6
var HTTPProtocolUnsupportedRegions = []string{
"us-iso-east-1",
"us-iso-west-1",
"us-isob-east-1",
"us-isob-west-1",
"us-isof-south-1",
"us-isof-east-1",
}

type EC2API interface {
DescribeCapacityReservations(context.Context, *ec2.DescribeCapacityReservationsInput, ...func(*ec2.Options)) (*ec2.DescribeCapacityReservationsOutput, error)
DescribeImages(context.Context, *ec2.DescribeImagesInput, ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error)
Expand Down
2 changes: 1 addition & 1 deletion pkg/cloudprovider/cloudprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim)
})
nc := c.instanceToNodeClaim(instance, instanceType, nodeClass)
nc.Annotations = lo.Assign(nc.Annotations, map[string]string{
v1.AnnotationEC2NodeClassHash: nodeClass.Hash(),
v1.AnnotationEC2NodeClassHash: nodeClass.HashForRegion(c.instanceProvider.Region()),
v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion,
v1.AnnotationInstanceProfile: nodeClass.Status.InstanceProfile,
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/controllers.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func NewControllers(
amiResolver amifamily.Resolver,
) []controller.Controller {
controllers := []controller.Controller{
nodeclasshash.NewController(kubeClient),
nodeclasshash.NewController(kubeClient, cfg.Region),
nodeclass.NewController(clk, kubeClient, cloudProvider, recorder, cfg.Region, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, instanceTypeProvider, launchTemplateProvider, capacityReservationProvider, ec2api, validationCache, recreationCache, amiResolver, options.FromContext(ctx).DisableDryRun),
nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
nodeclaimtagging.NewController(kubeClient, cloudProvider, instanceProvider),
Expand Down
8 changes: 5 additions & 3 deletions pkg/controllers/nodeclass/hash/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ import (

type Controller struct {
kubeClient client.Client
region string
}

func NewController(kubeClient client.Client) *Controller {
func NewController(kubeClient client.Client, region string) *Controller {
return &Controller{
kubeClient: kubeClient,
region: region,
}
}

Expand All @@ -55,7 +57,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass)
}
}
nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{
v1.AnnotationEC2NodeClassHash: nodeClass.Hash(),
v1.AnnotationEC2NodeClassHash: nodeClass.HashForRegion(c.region),
v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion,
})

Expand Down Expand Up @@ -103,7 +105,7 @@ func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1.EC2N
// Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed
if nc.StatusConditions().Get(karpv1.ConditionTypeDrifted) == nil {
nc.Annotations = lo.Assign(nc.Annotations, map[string]string{
v1.AnnotationEC2NodeClassHash: nodeClass.Hash(),
v1.AnnotationEC2NodeClassHash: nodeClass.HashForRegion(c.region),
})
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/nodeclass/hash/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ var _ = BeforeSuite(func() {
ctx = options.ToContext(ctx, test.Options())
awsEnv = test.NewEnvironment(ctx, env)

hashController = hash.NewController(env.Client)
hashController = hash.NewController(env.Client, "us-west-2")
})

var _ = AfterSuite(func() {
Expand Down
10 changes: 2 additions & 8 deletions pkg/providers/amifamily/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"sigs.k8s.io/karpenter/pkg/scheduling"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
awssdk "github.com/aws/karpenter-provider-aws/pkg/aws"
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap"
"github.com/aws/karpenter-provider-aws/pkg/providers/ssm"
)
Expand Down Expand Up @@ -278,14 +279,7 @@ func (r DefaultResolver) resolveLaunchTemplates(
if len(capacityReservationIDs) == 0 {
capacityReservationIDs = append(capacityReservationIDs, "")
}
httpProtocolUnsupportedRegions := sets.New[string](
"us-iso-east-1",
"us-iso-west-1",
"us-isob-east-1",
"us-isob-west-1",
"us-isof-south-1",
"us-isof-east-1",
)
httpProtocolUnsupportedRegions := sets.New(awssdk.HTTPProtocolUnsupportedRegions...)
return lo.Map(capacityReservationIDs, func(id string, _ int) *LaunchTemplate {
resolved := &LaunchTemplate{
Options: options,
Expand Down
5 changes: 5 additions & 0 deletions pkg/providers/instance/instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ type Provider interface {
List(context.Context) ([]*Instance, error)
Delete(context.Context, string) error
CreateTags(context.Context, string, map[string]string) error
Region() string
}

type options struct {
Expand Down Expand Up @@ -582,6 +583,10 @@ func instancesFromOutput(ctx context.Context, out *ec2.DescribeInstancesOutput)
return lo.Map(instances, func(i ec2types.Instance, _ int) *Instance { return NewInstance(ctx, i) }), nil
}

func (p *DefaultProvider) Region() string {
return p.region
}

func combineFleetErrors(fleetErrs []ec2types.CreateFleetError) (errs error) {
unique := sets.NewString()
for _, err := range fleetErrs {
Expand Down