Merge pull request #69 from cloudkite-io/add-spot-for-azure

add spot nodes configuration for aks

Author: kirill-cloudkite

standard-app/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: standard-app
 description: A Helm chart library by Cloudkite
 type: application
-version: 1.0.0
+version: 1.1.0
 maintainters:
   - email: [email protected]
     name: cloudkite

standard-app/example.values.yaml

@@ -12,6 +12,9 @@ labels:
   label1: somevalue
   label2: anothervalue
 
+# only aks is supported at the moment, you can add other options to _spot_node_selection.tpl
+preferSpot: aks
+
 ingress:  # NOTE: >0.8.0 DEPRECATES this, USE .Values.ingresses
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod

standard-app/templates/workloads/_spot_node_selection.tpl

@@ -0,0 +1,29 @@
+{{- define "standard-app.spotNodeSelection.nodeAffinity" -}}
+{{- if eq .Values.preferSpot "aks" }}
+nodeAffinity:
+  preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 60
+      preference:
+        matchExpressions:
+          - key: 'kubernetes.azure.com/scalesetpriority'
+            operator: In
+            values:
+              - 'spot'
+    - weight: 10
+      preference:
+        matchExpressions:
+          - key: 'kubernetes/nodetype'
+            operator: In
+            values:
+              - 'regular'
+{{- end -}}
+{{- end -}}
+
+{{- define "standard-app.spotNodeSelection.toleration" -}}
+{{- if eq .Values.preferSpot "aks" }}
+- key: kubernetes.azure.com/scalesetpriority
+  operator: Equal
+  value: spot
+  effect: NoSchedule
+{{- end -}}
+{{- end -}}

standard-app/templates/workloads/cronjob.yaml

@@ -4,7 +4,7 @@
       "Values"    $.Values
       "Release"   $.Release
       "labels"    (default (dict) $.Values.labels)
-    -}}
+    }}
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -19,18 +19,25 @@ spec:
       name: {{ $cronjobName }}
       {{- include "standard-app.labels" $cronjobValues | nindent 6 }}
     spec:
-      backoffLimit: {{ $cronjobConfig.backoffLImit | default "0" }}
+      backoffLimit: {{ $cronjobConfig.backoffLimit | default "0" }}
       template:
         metadata:
           {{- include "standard-app.labels" $cronjobValues | nindent 10 }}
         spec:
-          {{- if $cronjobConfig.affinity }}
+          {{- if or $cronjobConfig.affinity $.Values.preferSpot }}
           affinity:
-            {{- toYaml $cronjobConfig.affinity | nindent 12 }}
+            {{- $jobAffinity := default dict $cronjobConfig.affinity }}
+            {{- $spotAffinity := fromYaml (include "standard-app.spotNodeSelection.nodeAffinity" $ | trim) }}
+            {{- toYaml (merge $spotAffinity $jobAffinity) | nindent 10 }}
           {{- end }}
-          {{- if $cronjobConfig.tolerations }}
+          {{- if or $cronjobConfig.tolerations $.Values.preferSpot }}
           tolerations:
-            {{- toYaml $cronjobConfig.tolerations | nindent 12 }}
+            {{- if $cronjobConfig.tolerations }}
+              {{- toYaml $cronjobConfig.tolerations | nindent 12 }}
+            {{- end }}
+            {{- if $.Values.preferSpot }}
+              {{- include "standard-app.spotNodeSelection.toleration" $ | trim | nindent 12 }}
+            {{- end }}
           {{- end }}
           {{- if $cronjobConfig.nodeSelector }}
           nodeSelector:

standard-app/templates/workloads/deployment.yaml

@@ -36,13 +36,20 @@ spec:
         secret.reloader.stakater.com/reload: {{ $.Release.Name }}
         {{- end }}
     spec:
-      {{- if $appConfig.affinity }}
+      {{- if or $appConfig.affinity $.Values.preferSpot }}
       affinity:
-        {{- toYaml $appConfig.affinity | nindent 8 }}
+        {{- $jobAffinity := default dict $appConfig.affinity }}
+        {{- $spotAffinity := fromYaml (include "standard-app.spotNodeSelection.nodeAffinity" $ | trim) }}
+        {{- toYaml (merge $spotAffinity $jobAffinity) | nindent 8 }}
       {{- end }}
-      {{- if $appConfig.tolerations }}
+      {{- if or $appConfig.tolerations $.Values.preferSpot }}
       tolerations:
-        {{- toYaml $appConfig.tolerations | nindent 8 }}
+        {{- if $appConfig.tolerations }}
+          {{- toYaml $appConfig.tolerations | nindent 8 }}
+        {{- end }}
+        {{- if  $.Values.preferSpot }}
+          {{- include "standard-app.spotNodeSelection.toleration" $ | trim | nindent 8 }}
+        {{- end }}
       {{- end }}
       {{- if $appConfig.nodeSelector }}
       nodeSelector:

standard-app/templates/workloads/job.yaml

@@ -19,13 +19,20 @@ spec:
     metadata:
       {{- include "standard-app.labels" $jobValues | nindent 6 }}
     spec:
-      {{- if $jobConfig.affinity }}
+      {{- if or $jobConfig.affinity $.Values.preferSpot }}
       affinity:
-        {{- toYaml $jobConfig.affinity | nindent 8 }}
+        {{- $jobAffinity := default dict $jobConfig.affinity }}
+        {{- $spotAffinity := fromYaml (include "standard-app.spotNodeSelection.nodeAffinity" $ | trim) }}
+        {{- toYaml (merge $spotAffinity $jobAffinity) | nindent 8 }}
       {{- end }}
-      {{- if $jobConfig.tolerations }}
+      {{- if or $jobConfig.tolerations $.Values.preferSpot }}
       tolerations:
-        {{- toYaml $jobConfig.tolerations | nindent 8 }}
+        {{- if $jobConfig.tolerations }}
+          {{- toYaml $jobConfig.tolerations | nindent 8 }}
+        {{- end }}
+        {{- if $.Values.preferSpot }}
+          {{- include "standard-app.spotNodeSelection.toleration" $ | trim | nindent 8 }}
+        {{- end }}
       {{- end }}
       {{- if $jobConfig.nodeSelector }}
       nodeSelector: