Repository navigation
Expand file tree
/
Copy pathdolt-failover.yaml
More file actions
73 lines (71 loc) · 2.25 KB
/
Copy pathdolt-failover.yaml
File metadata and controls
73 lines (71 loc) · 2.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
apiVersion: batch/v1
kind: CronJob
metadata:
name: dolt-failover-job
namespace: dolt-cluster-example
spec:
schedule: "@yearly" # Very infrequent
suspend: true # Prevents automatic execution
jobTemplate:
spec:
backoffLimit: 0
template:
spec:
serviceAccountName: doltclusterctl
containers:
- name: doltclusterctl
image: priley86/doltclusterctl:arm64
imagePullPolicy: Always
env:
- name: DOLT_USERNAME
valueFrom:
secretKeyRef:
name: dolt-credentials
key: admin-user
- name: DOLT_PASSWORD
valueFrom:
secretKeyRef:
name: dolt-credentials
key: admin-password
command:
- /bin/sh
- -c
- |
set -e
echo "[*] Running doltclusterctl promotestandby"
doltclusterctl -n dolt-cluster-example promotestandby dolt
echo "[✓] Failover complete."
restartPolicy: Never
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: dolt-failover-monitor
namespace: dolt-cluster-example
spec:
schedule: "*/1 * * * *" # Every minute
jobTemplate:
spec:
template:
spec:
serviceAccountName: doltclusterctl
containers:
- name: healthcheck
image: busybox
command:
- /bin/sh
- -c
- |
if ! timeout 5 nc -z dolt 3306; then
timestamp=$(date +%Y%m%d%H%M%S)
job_name="dolt-failover-triggered-$timestamp"
echo "Dolt primary unhealthy. Triggering failover..."
kubectl create job --from=cronjob/dolt-failover-job $job_name -n dolt-cluster-example
# Wait for the failover Job to complete (60s timeout)
kubectl wait --for=condition=complete job/$job_name -n dolt-cluster-example --timeout=60s
echo "[*] Restarting proxysql"
kubectl rollout restart deployment proxysql -n dolt-cluster-example
else
echo "Dolt primary is healthy."
fi
restartPolicy: OnFailure