Refactor K8s deployment for improved DRY configuration and flexibility (Deployment and StatefulSet strategies)

.gitignore

@@ -362,3 +362,4 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+/.claude

Directory.Build.props

@@ -4,12 +4,14 @@
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
-
   <PropertyGroup>
     <Copyright>Copyright © 2015-$([System.DateTime]::Now.Year) Petabridge</Copyright>
     <Authors>Petabridge</Authors>
-    <VersionPrefix>0.2.10</VersionPrefix>
-    <PackageReleaseNotes>Add TLS support for Akka.Remote in K8s deployment</PackageReleaseNotes>
+    <VersionPrefix>0.2.11</VersionPrefix>
+    <PackageReleaseNotes>- Added Deployment-based Kubernetes overlay as alternative to StatefulSet for reduced rebalancing overhead
+- Refactored K8s configuration to be DRY - split base resources into separate files
+- Added `-Strategy` parameter to deploy.ps1 to choose between StatefulSet and Deployment strategies
+- Fixed deployment script error handling for better reliability</PackageReleaseNotes>
     <PackageIconUrl>
     </PackageIconUrl>
     <PackageProjectUrl>
@@ -18,10 +20,8 @@
     </PackageLicenseUrl>
     <NoWarn>$(NoWarn);CS1591</NoWarn>
   </PropertyGroup>
-
   <ItemGroup Condition=" '$(CI)' == 'true' ">
     <ContainerLabel Include="com.docker.extension.changelog" Value="$(GITHUB_SERVER_URL)/$(GITHUB_REPOSITORY)/commit/$(GITHUB_SHA)" />
     <ContainerLabel Include="com.docker.extension.publisher-url" Value="$(GITHUB_SERVER_URL)/$(GITHUB_REPOSITORY_OWNER)" />
   </ItemGroup>
-
 </Project>

README.md

@@ -55,7 +55,9 @@ The version number is pulled from `Directory.Build.props` and automatically appl
 The K8s manifests use [Kustomize](https://kustomize.io/) for configuration management:
 
 * **Base configuration** (`k8s/base/`): Contains all core Kubernetes manifests without version tags
-* **Environment overlays** (`k8s/overlays/local/`): Environment-specific configurations that apply version tags
+* **Environment overlays**:
+  - `k8s/overlays/local/`: StatefulSet deployment with stable pod names
+  - `k8s/overlays/deployment/`: Deployment-based configuration with reduced rebalancing overhead
 
 ### Deployment Scripts
 
@@ -64,15 +66,24 @@ The K8s manifests use [Kustomize](https://kustomize.io/) for configuration manag
 To deploy or update all services to your local Kubernetes cluster:
 
 ```powershell
-./k8s/deployAll.ps1
+# Deploy with StatefulSet (default)
+./k8s/deploy.ps1
+
+# Deploy with Deployment strategy (reduced rebalancing during updates)
+./k8s/deploy.ps1 -Strategy deployment
 ```
 
 This script will:
 1. Extract the version from `Directory.Build.props`
-2. Update the Kustomize overlay with the current version
-3. Create the `drawtogether` namespace if needed
-4. Apply all Kubernetes manifests using Kustomize
-5. Wait for the StatefulSet rollout to complete (with zero-downtime rolling updates)
+2. Build Docker images if they don't exist for the current version
+3. Update the Kustomize overlay with the current version
+4. Create the `drawtogether` namespace if needed
+5. Apply all Kubernetes manifests using Kustomize
+6. Wait for the rollout to complete (with zero-downtime rolling updates)
+
+**Deployment Strategies:**
+- **statefulset** (default): Stable pod names, predictable scaling, more rebalancing during updates
+- **deployment**: Dynamic pod names, surge updates, less rebalancing overhead
 
 **This script is idempotent** - run it for both initial deployment and updates. Kubernetes automatically performs zero-downtime rolling updates when you change the image version.
 

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+#### 0.2.11 October 23rd 2024 ####
+
+- Added Deployment-based Kubernetes overlay as alternative to StatefulSet for reduced rebalancing overhead
+- Refactored K8s configuration to be DRY - split base resources into separate files
+- Added `-Strategy` parameter to deploy.ps1 to choose between StatefulSet and Deployment strategies
+- Fixed deployment script error handling for better reliability
+
 #### 0.2.2 May 2nd 2024 ####
 
 Added support for loading configuration via Msft.Ext-compatible environment variables, `appSettings.json`, and `appSettings.{ASPNETCORE_ENVIRONMENT}.json`

k8s/base/ingress.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: drawtogether-localhost
+  namespace: drawtogether
+  annotations:
+    kubernetes.io/ingress.class: "nginx"  # Updated annotation
+    nginx.ingress.kubernetes.io/affinity: "cookie"
+    nginx.ingress.kubernetes.io/session-cookie-name: "route"
+    nginx.ingress.kubernetes.io/session-cookie-hash: "sha1"
+spec:
+  rules:
+  - host: drawtogether.localdev.me
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: drawtogether
+            port:
+              number: 8080

k8s/base/kustomization.yaml

@@ -7,4 +7,6 @@ resources:
   - sql-secret.yaml
   - sql-server.yaml
   - k8s-migrations-job.yaml
-  - k8s-web-service.yaml
+  - ingress.yaml
+  - service.yaml
+  - statefulset.yaml

k8s/overlays/deployment/service.yaml → k8s/base/service.yaml

@@ -6,10 +6,7 @@ metadata:
   labels:
     app: drawtogether
 spec:
-  # For Deployment, we use a regular Service (not headless)
-  # publishNotReadyAddresses is still important for Akka.Discovery
   publishNotReadyAddresses: true
-  # We still use ClusterIP: None for discovery purposes
   clusterIP: None
   ports:
     - port: 8558
@@ -19,4 +16,4 @@ spec:
     - port: 8080
       name: http
   selector:
-    app: drawtogether
+    app: drawtogether

k8s/base/k8s-web-service.yaml → k8s/base/statefulset.yaml

@@ -1,46 +1,3 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: drawtogether-localhost
-  namespace: drawtogether
-  annotations:
-    kubernetes.io/ingress.class: "nginx"  # Updated annotation
-    nginx.ingress.kubernetes.io/affinity: "cookie"
-    nginx.ingress.kubernetes.io/session-cookie-name: "route"
-    nginx.ingress.kubernetes.io/session-cookie-hash: "sha1"
-spec:
-  rules:
-  - host: drawtogether.localdev.me
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: drawtogether
-            port:
-              number: 8080
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: drawtogether
-  namespace: drawtogether
-  labels:
-    app: drawtogether
-spec:
-  publishNotReadyAddresses: true
-  clusterIP: None
-  ports:
-    - port: 8558
-      name: management
-    - port: 5055
-      name: akka-remote
-    - port: 8080
-      name: http
-  selector:
-    app: drawtogether
----
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -69,9 +26,20 @@ spec:
           - /bin/bash
           - -c
           - |
-            echo "Waiting for migrations job to complete..."
-            kubectl wait --for=condition=complete --timeout=300s job/drawtogether-migrations -n drawtogether
-            echo "Migrations completed successfully!"
+            echo "Checking migrations status..."
+            # Check if job exists
+            if kubectl get job drawtogether-migrations -n drawtogether &>/dev/null; then
+              # Job exists, check if it's already completed
+              if kubectl get job drawtogether-migrations -n drawtogether -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' | grep -q "True"; then
+                echo "Migrations job already completed"
+              else
+                echo "Waiting for migrations job to complete..."
+                kubectl wait --for=condition=complete --timeout=300s job/drawtogether-migrations -n drawtogether
+                echo "Migrations completed successfully!"
+              fi
+            else
+              echo "Migrations job not found (likely already completed and cleaned up)"
+            fi
       containers:
       - name: pbm-sidecar
         image: petabridge/pbm:latest #sidecar
@@ -154,4 +122,4 @@ spec:
       volumes:
       - name: tls-cert
         secret:
-          secretName: akka-tls-cert
+          secretName: akka-tls-cert