Merge pull request #91 from kubeflow-onprem/install-ccm-and-kube-vip

Install ccm and kube vip

Author: displague

.gitignore

@@ -1,4 +1,4 @@
-.terraform/*
+**/.terraform/*
 *.tfstate
 *.tfstate.backup
 *.tfvars

.terraform.lock.hcl

@@ -54,4 +54,6 @@ module "controllers" {
   skip_workloads           = var.skip_workloads
   control_plane_node_count = var.control_plane_node_count
   ssh_private_key_path     = local_file.cluster_private_key_pem.filename
+  ccm_enabled              = var.ccm_enabled
+  loadbalancer_type        = var.loadbalancer_type
 }

kubernetes-controller-pool.tf

@@ -13,6 +13,8 @@ module "node_pool_blue" {
   controller_address = module.controllers.controller_addresses
   project_id         = var.metal_create_project ? metal_project.new_project[0].id : var.project_id
   storage            = var.storage
+  ccm_enabled        = var.ccm_enabled
+
 }
 
 module "node_pool_gpu_green" {
@@ -28,4 +30,5 @@ module "node_pool_gpu_green" {
   controller_address = module.controllers.controller_addresses
   project_id         = var.metal_create_project ? metal_project.new_project[0].id : var.project_id
   storage            = var.storage
+  ccm_enabled        = var.ccm_enabled
 }

kubernetes-node-pool.tf

@@ -82,14 +82,14 @@ function gpu_config {
 }
 
 function metal_lb {
-    echo "Configuring MetalLB for ${metal_network_cidr}..." && \
+  echo "Configuring MetalLB for ${metal_network_cidr}..." && \
     cd $HOME/kube ; \
     cat << EOF > metal_lb.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  namespace: metallb-system
-  name: config
+  namespace: ${metallb_namespace}
+  name: ${metallb_configmap}
 data:
   config: |
     address-pools:
@@ -98,6 +98,27 @@ data:
       addresses:
       - ${metal_network_cidr}
 EOF
+
+  echo "Applying MetalLB manifests..." && \
+    cd $HOME/kube && \
+    kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f $(cat $HOME/workloads.json | jq .metallb_namespace) && \
+    kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f $(cat $HOME/workloads.json | jq .metallb_release) && \
+    kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" && \
+    kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f metal_lb.yaml
+}
+
+function kube_vip {
+  kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+  GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway");
+  ip route add 169.254.255.1 via $GATEWAY_IP
+  ip route add 169.254.255.2 via $GATEWAY_IP
+  alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.3.8"
+  kube-vip manifest daemonset \
+  --interface lo \
+  --services \
+  --bgp \
+  --annotations metal.equinix.com \
+  --inCluster | kubectl apply -f -
 }
 
 function ceph_pre_check {
@@ -107,7 +128,7 @@ function ceph_pre_check {
 
 function ceph_rook_basic {
   cd $HOME/kube ; \
-  mkdir ceph ;\ 
+  mkdir ceph ;\
   echo "Pulled Manifest for Ceph-Rook..." && \
   kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f $(cat $HOME/workloads.json | jq .ceph_common) ; \
   sleep 30 ; \
@@ -177,15 +198,6 @@ acert="/etc/kubernetes/pki/etcd/ca.crt" get /registry/secrets/default/personal-s
   sed -i 's|    volumeMounts:|    volumeMounts:\n    - mountPath: /etc/kubernetes/secrets.conf\n      name: secretconfig\n      readOnly: true|g' /etc/kubernetes/manifests/kube-apiserver.yaml 
 }
 
-function apply_workloads {
-  echo "Applying workloads..." && \
-	cd $HOME/kube && \
-	kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f $(cat $HOME/workloads.json | jq .metallb_namespace) && \
-	kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f $(cat $HOME/workloads.json | jq .metallb_release) && \
-	kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" && \
-  kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f metal_lb.yaml
-}
-
 function apply_extra {
   workload_manifests=$(cat $HOME/workloads.json | jq .extra | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | tr , '\n') && \
   if [ "$workload_manifests" == "" ]; then
@@ -197,11 +209,35 @@ function apply_extra {
   fi
 }
 
+function install_ccm {
+  cat << EOF > $HOME/kube/equinix-ccm-config.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metal-cloud-config
+  namespace: kube-system
+stringData:
+  cloud-sa.json: |
+    {
+      "apiKey": "${equinix_api_key}",
+      "projectID": "${equinix_project_id}",
+      "loadbalancer": "${loadbalancer}"
+    }
+EOF
+
+kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f $HOME/kube/equinix-ccm-config.yaml
+RELEASE=${ccm_version}
+kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://github.com/equinix/cloud-provider-equinix-metal/releases/download/$RELEASE/deployment.yaml
+}
+
 install_docker && \
 enable_docker && \
 load_workloads && \
 install_kube_tools && \
 sleep 30 && \
+if [ "${ccm_enabled}" = "true" ]; then
+  echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet
+fi
 if [ "${control_plane_node_count}" = "0" ]; then
   echo "No control plane nodes provisioned, initializing single master..." ; \
   init_cluster
@@ -212,8 +248,16 @@ fi
 
 sleep 180 && \
 configure_network
-metal_lb && \
-apply_workloads
+if [ "${ccm_enabled}" = "true" ]; then
+install_ccm
+sleep 30 # The CCM will probably take a while to reconcile
+fi
+if [ "${loadbalancer_type}" = "metallb" ]; then
+metal_lb
+fi
+if [ "${loadbalancer_type}" = "kube-vip" ]; then
+kube_vip
+fi
 if [ "${count_gpu}" = "0" ]; then
   echo "Skipping GPU enable..."
 else

modules/controller_pool/.terraform.lock.hcl

@@ -39,4 +39,4 @@ enable_docker && \
 install_kube_tools && \
 sleep 180 ; \
 backoff_count=`echo $((5 + RANDOM % 100))` ; \
-sleep $backoff_count
+sleep $backoff_count # Shouldn't there be a kubeadm join command somewhere? Looks like we just install tools and do nothing else

modules/controller_pool/controller-primary.tpl

@@ -0,0 +1,4 @@
+locals {
+  // Only support MetalLB and Kube-VIP
+  loadbalancer_config = var.loadbalancer_type == "metallb" ? "metallb:///${var.metallb_namespace}/${var.metallb_configmap}" : "kube-vip://"
+}

modules/controller_pool/controller-standby.tpl

@@ -15,6 +15,14 @@ data "template_file" "controller-primary" {
     skip_workloads           = var.skip_workloads ? "yes" : "no"
     workloads                = jsonencode(var.workloads)
     control_plane_node_count = var.control_plane_node_count
+    equinix_api_key          = var.auth_token
+    equinix_project_id       = var.project_id
+    loadbalancer             = local.loadbalancer_config
+    loadbalancer_type        = var.loadbalancer_type
+    ccm_version              = var.ccm_version
+    ccm_enabled              = var.ccm_enabled
+    metallb_namespace        = var.metallb_namespace
+    metallb_configmap        = var.metallb_configmap
   }
 }