Merge pull request #8625 from wenxuan0923/wenx/hobo-1.34

[AKS HOBO][1.34 release] Add support for hosted-on-behalf-of systempool autoscaling

Author: k8s-ci-robot

cluster-autoscaler/cloudprovider/azure/azure_cache.go

@@ -107,11 +107,17 @@ type azureCache struct {
 }
 
 func newAzureCache(client *azClient, cacheTTL time.Duration, config Config) (*azureCache, error) {
+	nodeResourceGroup := config.ResourceGroup
+	// Hosted (on-behalf-of) system pool node resources are in the AKS internal resource group within AME tenants,
+	// which differs from the MC_* resource group found in the customer subscription.
+	if config.HostedResourceGroup != "" {
+		nodeResourceGroup = config.HostedResourceGroup
+	}
 	cache := &azureCache{
 		interrupt:            make(chan struct{}),
 		azClient:             client,
 		refreshInterval:      cacheTTL,
-		resourceGroup:        config.ResourceGroup,
+		resourceGroup:        nodeResourceGroup,
 		clusterResourceGroup: config.ClusterResourceGroup,
 		clusterName:          config.ClusterName,
 		enableVMsAgentPool:   config.EnableVMsAgentPool,

cluster-autoscaler/cloudprovider/azure/azure_config.go

@@ -65,6 +65,15 @@ type Config struct {
 	// It can override the default public ARM endpoint for VMs pool scale operations.
 	ARMBaseURLForAPClient string `json:"armBaseURLForAPClient" yaml:"armBaseURLForAPClient"`
 
+	// Hosted (on-behalf-of) system pool configuration for automatic cluster.
+	// HostedSubscriptionID is the subscription ID of the hosted resources under AKS internal tenant.
+	HostedSubscriptionID string `json:"hostedSubscriptionID" yaml:"hostedSubscriptionID"`
+	// HostedResourceGroup is the resource group of the hosted resources under AKS internal tenant.
+	HostedResourceGroup string `json:"hostedResourceGroup" yaml:"hostedResourceGroup"`
+	// HostedResourceProxyURL is the URL to use for retrieving hosted resources under AKS internal tenant.
+	// It can override the default public ARM endpoint for operations like VM/SKU GET.
+	HostedResourceProxyURL string `json:"hostedResourceProxyURL" yaml:"hostedResourceProxyURL"`
+
 	// AuthMethod determines how to authorize requests for the Azure
 	// cloud. Valid options are "principal" (= the traditional
 	// service principle approach) and "cli" (= load az command line
@@ -223,6 +232,15 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) {
 	if _, err = assignFromEnvIfExists(&cfg.SubscriptionID, "ARM_SUBSCRIPTION_ID"); err != nil {
 		return nil, err
 	}
+	if _, err = assignFromEnvIfExists(&cfg.HostedResourceProxyURL, "HOSTED_RESOURCE_PROXY_URL"); err != nil {
+		return nil, err
+	}
+	if _, err = assignFromEnvIfExists(&cfg.HostedSubscriptionID, "HOSTED_SUBSCRIPTION_ID"); err != nil {
+		return nil, err
+	}
+	if _, err = assignFromEnvIfExists(&cfg.HostedResourceGroup, "HOSTED_RESOURCE_GROUP"); err != nil {
+		return nil, err
+	}
 	if _, err = assignBoolFromEnvIfExists(&cfg.UseManagedIdentityExtension, "ARM_USE_MANAGED_IDENTITY_EXTENSION"); err != nil {
 		return nil, err
 	}
@@ -380,6 +398,17 @@ func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azu
 		}
 	}
 
+	// A proxy service is required to access resources for the Hosted (on-behalf-of) system pool within automatic clusters.
+	if cfg.HostedResourceProxyURL != "" {
+		azClientConfig.ResourceManagerEndpoint = cfg.HostedResourceProxyURL
+	}
+
+	// Hosted (on-behalf-of) system pool resources are hosted under AKS internal tenant and subscription.
+	// it is different from the customer subscription where the cluster is created.
+	if cfg.HostedSubscriptionID != "" {
+		azClientConfig.SubscriptionID = cfg.HostedSubscriptionID
+	}
+
 	if cfg.HasExtendedLocation() {
 		azClientConfig.ExtendedLocation = &azclients.ExtendedLocation{
 			Name: cfg.ExtendedLocationName,

cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go

@@ -154,9 +154,16 @@ func (vmPool *VMPool) IncreaseSize(delta int) error {
 	if len(versionedAP.Properties.VirtualMachinesProfile.Scale.Manual) > 0 {
 		requestBody = buildRequestBodyForScaleUp(versionedAP, count, vmPool.sku)
 
-	} else { // AKS-managed CAS will use custom header for setting the target count
+	}
+	// hosted CAS will be using Autoscale scale profile
+	// HostedSystem will be using manual scale profile
+	// Both of them need to set the Target-Count and SKU headers
+	if len(versionedAP.Properties.VirtualMachinesProfile.Scale.Autoscale) > 0 ||
+		(versionedAP.Properties.Mode != nil &&
+			strings.EqualFold(string(*versionedAP.Properties.Mode), "HostedSystem")) {
 		header := make(http.Header)
 		header.Set("Target-Count", fmt.Sprintf("%d", count))
+		header.Set("SKU", fmt.Sprintf("%s", vmPool.sku))
 		updateCtx = policy.WithHTTPHeader(updateCtx, header)
 	}