Skip to content

Commit 605c898

Browse files
johscheuerjohscheuer
authored
Improve the backup e2e tests to allow multiple tests (#2364)
* Improve the backup e2e tests to allow multiple tests * Add finalizer removal in cleanup script * Correct and enable the partionted log backup system * Correct operator backup tests for restorable version * Disable restore test with restorable version
1 parent cb69e09 commit 605c898

File tree

8 files changed

+201
-79
lines changed

8 files changed

+201
-79
lines changed

‎e2e/fixtures/factory.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,13 @@ func (factory *Factory) CreateIfAbsent(object client.Object) error {
784784
objectCopy,
785785
)
786786

787+
if objectCopy.GetDeletionTimestamp() != nil {
788+
return fmt.Errorf(
789+
"resource is currently under deletion with deletion timestamp: %s",
790+
objectCopy.GetDeletionTimestamp().String(),
791+
)
792+
}
793+
787794
if err != nil {
788795
if k8serrors.IsNotFound(err) {
789796
return ctrlClient.Create(context.Background(), object)

‎e2e/fixtures/fdb_backup.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,19 @@ func (fdbBackup *FdbBackup) WaitForRestorableVersion(version uint64) uint64 {
235235
status := &fdbv1beta2.FoundationDBLiveBackupStatusState{}
236236
g.Expect(json.Unmarshal([]byte(out), status)).NotTo(gomega.HaveOccurred())
237237

238-
log.Println("Backup status:", status)
238+
var latestRestorableVersion uint64
239+
if status.LatestRestorablePoint != nil {
240+
latestRestorableVersion = ptr.Deref(status.LatestRestorablePoint.Version, 0)
241+
}
242+
243+
log.Println(
244+
"Backup status running:",
245+
status.Running,
246+
"restorable:",
247+
ptr.Deref(status.Restorable, false),
248+
"latestRestorablePoint:",
249+
latestRestorableVersion,
250+
)
239251
g.Expect(ptr.Deref(status.Restorable, false)).To(gomega.BeTrue())
240252
g.Expect(status.LatestRestorablePoint).NotTo(gomega.BeNil())
241253

@@ -273,5 +285,13 @@ func (fdbBackup *FdbBackup) Destroy() {
273285
}
274286

275287
g.Expect(err).NotTo(gomega.HaveOccurred())
288+
}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
289+
290+
// Ensure that the resource is removed.
291+
gomega.Eventually(func(g gomega.Gomega) {
292+
backup := &fdbv1beta2.FoundationDBBackup{}
293+
err := fdbBackup.fdbCluster.getClient().
294+
Get(context.Background(), client.ObjectKeyFromObject(fdbBackup.backup), backup)
295+
g.Expect(k8serrors.IsNotFound(err)).To(gomega.BeTrue())
276296
}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
277297
}

‎e2e/fixtures/fdb_cluster.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,10 +1740,10 @@ func (fdbCluster *FdbCluster) ClearRange(prefixBytes []byte, timeout int) {
17401740
gomega.Expect(err).NotTo(gomega.HaveOccurred())
17411741
end := FdbPrintable(endBytes)
17421742
_, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(fmt.Sprintf(
1743-
"writemode on; clearrange %s %s",
1743+
"writemode on; option on ACCESS_SYSTEM_KEYS; clearrange %s %s",
17441744
begin,
17451745
end,
1746-
), false, timeout)
1746+
), true, timeout)
17471747

17481748
gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr)
17491749
}
@@ -1811,7 +1811,7 @@ func (fdbCluster *FdbCluster) GenerateRandomValues(
18111811
for i := 0; i < n; i++ {
18121812
res = append(res, KeyValue{
18131813
Key: append([]byte{prefix}, index...),
1814-
Value: []byte(fdbCluster.factory.RandStringRunes(4)),
1814+
Value: []byte(fdbCluster.factory.RandStringRunes(24)),
18151815
})
18161816
index, err = FdbStrinc(index)
18171817
gomega.Expect(err).NotTo(gomega.HaveOccurred())

‎e2e/fixtures/fdb_restore.go‎

Lines changed: 64 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -26,61 +26,75 @@ import (
2626
"strconv"
2727
"time"
2828

29+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
30+
2931
"sigs.k8s.io/controller-runtime/pkg/client"
3032

3133
fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/v2/api/v1beta2"
3234
"github.com/onsi/gomega"
3335
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3436
)
3537

38+
// FdbRestore represents a fdbv1beta2.FoundationDBRestore resource for doing restores to a FdbCluster.
39+
type FdbRestore struct {
40+
restore *fdbv1beta2.FoundationDBRestore
41+
fdbCluster *FdbCluster
42+
}
43+
3644
// CreateRestoreForCluster will create a FoundationDBRestore resource based on the provided backup resource.
3745
// For more information how the backup system with the operator is working please look at
3846
// the operator documentation: https://github.com/FoundationDB/fdb-kubernetes-operator/v2/blob/master/docs/manual/backup.md
39-
func (factory *Factory) CreateRestoreForCluster(backup *FdbBackup, backupVersion *uint64) {
47+
func (factory *Factory) CreateRestoreForCluster(
48+
backup *FdbBackup,
49+
backupVersion *uint64,
50+
) *FdbRestore {
4051
gomega.Expect(backup).NotTo(gomega.BeNil())
41-
restore := &fdbv1beta2.FoundationDBRestore{
42-
ObjectMeta: metav1.ObjectMeta{
43-
Name: backup.fdbCluster.Name(),
44-
Namespace: backup.fdbCluster.Namespace(),
52+
restore := &FdbRestore{
53+
restore: &fdbv1beta2.FoundationDBRestore{
54+
ObjectMeta: metav1.ObjectMeta{
55+
Name: backup.fdbCluster.Name(),
56+
Namespace: backup.fdbCluster.Namespace(),
57+
},
58+
Spec: fdbv1beta2.FoundationDBRestoreSpec{
59+
DestinationClusterName: backup.fdbCluster.Name(),
60+
BlobStoreConfiguration: backup.backup.Spec.BlobStoreConfiguration,
61+
CustomParameters: backup.backup.Spec.CustomParameters,
62+
BackupVersion: backupVersion,
63+
},
4564
},
46-
Spec: fdbv1beta2.FoundationDBRestoreSpec{
47-
DestinationClusterName: backup.fdbCluster.Name(),
48-
BlobStoreConfiguration: backup.backup.Spec.BlobStoreConfiguration,
49-
CustomParameters: backup.backup.Spec.CustomParameters,
50-
},
51-
}
52-
53-
if backupVersion != nil {
54-
restore.Spec.BackupVersion = backupVersion
65+
fdbCluster: backup.fdbCluster,
5566
}
5667

57-
gomega.Expect(factory.CreateIfAbsent(restore)).NotTo(gomega.HaveOccurred())
68+
gomega.Expect(factory.CreateIfAbsent(restore.restore)).NotTo(gomega.HaveOccurred())
5869

5970
factory.AddShutdownHook(func() error {
60-
return factory.GetControllerRuntimeClient().Delete(context.Background(), restore)
71+
restore.Destroy()
72+
return nil
6173
})
6274

63-
waitForRestoreToComplete(backup)
75+
restore.waitForRestoreToComplete(backup)
76+
77+
return restore
6478
}
6579

6680
// waitForRestoreToComplete waits until the restore completed.
67-
func waitForRestoreToComplete(backup *FdbBackup) {
68-
ctrlClient := backup.fdbCluster.getClient()
81+
func (restore *FdbRestore) waitForRestoreToComplete(backup *FdbBackup) {
82+
ctrlClient := restore.fdbCluster.getClient()
6983

7084
lastReconcile := time.Now()
7185
gomega.Eventually(func(g gomega.Gomega) fdbv1beta2.FoundationDBRestoreState {
72-
restore := &fdbv1beta2.FoundationDBRestore{}
73-
g.Expect(ctrlClient.Get(context.Background(), client.ObjectKeyFromObject(backup.backup), restore)).
86+
currentRestore := &fdbv1beta2.FoundationDBRestore{}
87+
g.Expect(ctrlClient.Get(context.Background(), client.ObjectKeyFromObject(restore.restore), currentRestore)).
7488
To(gomega.Succeed())
75-
log.Println("restore state:", restore.Status.State)
89+
log.Println("restore state:", currentRestore.Status.State)
7690

7791
if time.Since(lastReconcile) > time.Minute {
7892
lastReconcile = time.Now()
79-
patch := client.MergeFrom(restore.DeepCopy())
80-
if restore.Annotations == nil {
81-
restore.Annotations = make(map[string]string)
93+
patch := client.MergeFrom(currentRestore.DeepCopy())
94+
if currentRestore.Annotations == nil {
95+
currentRestore.Annotations = make(map[string]string)
8296
}
83-
restore.Annotations["foundationdb.org/reconcile"] = strconv.FormatInt(
97+
currentRestore.Annotations["foundationdb.org/reconcile"] = strconv.FormatInt(
8498
time.Now().UnixNano(),
8599
10,
86100
)
@@ -89,10 +103,10 @@ func waitForRestoreToComplete(backup *FdbBackup) {
89103
// This should speed up the reconcile phase.
90104
gomega.Expect(ctrlClient.Patch(
91105
context.Background(),
92-
restore,
106+
currentRestore,
93107
patch)).To(gomega.Succeed())
94108

95-
out, _, err := backup.fdbCluster.ExecuteCmdOnPod(
109+
out, _, err := restore.fdbCluster.ExecuteCmdOnPod(
96110
*backup.GetBackupPod(),
97111
fdbv1beta2.MainContainerName,
98112
"fdbrestore status --dest_cluster_file $FDB_CLUSTER_FILE",
@@ -103,6 +117,27 @@ func waitForRestoreToComplete(backup *FdbBackup) {
103117
g.Expect(err).To(gomega.Succeed())
104118
}
105119

106-
return restore.Status.State
120+
return currentRestore.Status.State
107121
}).WithTimeout(20 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Equal(fdbv1beta2.CompletedFoundationDBRestoreState))
108122
}
123+
124+
// Destroy will delete the FoundationDBRestore for the associated FdbBackup if it exists.
125+
func (restore *FdbRestore) Destroy() {
126+
gomega.Eventually(func(g gomega.Gomega) {
127+
err := restore.fdbCluster.factory.GetControllerRuntimeClient().
128+
Delete(context.Background(), restore.restore)
129+
if k8serrors.IsNotFound(err) {
130+
return
131+
}
132+
133+
g.Expect(err).NotTo(gomega.HaveOccurred())
134+
}).WithTimeout(4 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
135+
136+
// Ensure that the resource is removed.
137+
gomega.Eventually(func(g gomega.Gomega) {
138+
currentRestore := &fdbv1beta2.FoundationDBRestore{}
139+
err := restore.fdbCluster.getClient().
140+
Get(context.Background(), client.ObjectKeyFromObject(restore.restore), currentRestore)
141+
g.Expect(k8serrors.IsNotFound(err)).To(gomega.BeTrue())
142+
}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
143+
}

‎e2e/fixtures/kubernetes_fixtures.go‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,21 @@
2121
package fixtures
2222

2323
import (
24-
ctx "context"
24+
"context"
2525
"fmt"
2626
"log"
2727
"time"
2828

29-
k8serrors "k8s.io/apimachinery/pkg/api/errors"
30-
31-
"k8s.io/apimachinery/pkg/runtime"
32-
"sigs.k8s.io/controller-runtime/pkg/client"
33-
"sigs.k8s.io/controller-runtime/pkg/client/config"
34-
29+
fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/v2/api/v1beta2"
3530
"github.com/onsi/gomega"
3631
corev1 "k8s.io/api/core/v1"
3732
rbacv1 "k8s.io/api/rbac/v1"
33+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
3834
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35+
"k8s.io/apimachinery/pkg/runtime"
36+
"k8s.io/utils/ptr"
37+
"sigs.k8s.io/controller-runtime/pkg/client"
38+
"sigs.k8s.io/controller-runtime/pkg/client/config"
3939
)
4040

4141
const (
@@ -108,7 +108,7 @@ func (factory *Factory) createNamespace(suffix string) string {
108108

109109
namespaceResource := &corev1.Namespace{}
110110
err = factory.controllerRuntimeClient.Get(
111-
ctx.Background(),
111+
context.Background(),
112112
client.ObjectKey{Namespace: "", Name: namespace},
113113
namespaceResource,
114114
)
@@ -162,16 +162,14 @@ func (factory *Factory) createNamespace(suffix string) string {
162162
factory.AddShutdownHook(func() error {
163163
log.Printf("finished all tests, start deleting namespace %s\n", namespace)
164164

165-
gomega.Eventually(func() error {
165+
gomega.Eventually(func(g gomega.Gomega) {
166166
podList := &corev1.PodList{}
167167
err := factory.controllerRuntimeClient.List(
168-
ctx.Background(),
168+
context.Background(),
169169
podList,
170170
client.InNamespace(namespace),
171171
)
172-
if err != nil {
173-
return err
174-
}
172+
g.Expect(err).NotTo(gomega.HaveOccurred())
175173

176174
for _, pod := range podList.Items {
177175
if len(pod.Finalizers) > 0 {
@@ -180,8 +178,40 @@ func (factory *Factory) createNamespace(suffix string) string {
180178
}
181179
}
182180

183-
return nil
184-
}).WithTimeout(2 * time.Minute).WithPolling(1 * time.Second).ShouldNot(gomega.HaveOccurred())
181+
backupList := &fdbv1beta2.FoundationDBBackupList{}
182+
err = factory.controllerRuntimeClient.List(
183+
context.Background(),
184+
backupList,
185+
client.InNamespace(namespace),
186+
)
187+
g.Expect(err).NotTo(gomega.HaveOccurred())
188+
189+
for _, backup := range backupList.Items {
190+
if len(backup.Finalizers) > 0 {
191+
log.Printf("Removing finalizer from backup %s/%s\n", namespace, backup.Name)
192+
gomega.Eventually(func(g gomega.Gomega) {
193+
fetchedBackup := &fdbv1beta2.FoundationDBBackup{}
194+
err = factory.controllerRuntimeClient.Get(
195+
context.Background(),
196+
client.ObjectKeyFromObject(ptr.To(backup)),
197+
fetchedBackup,
198+
)
199+
if k8serrors.IsNotFound(err) {
200+
return
201+
}
202+
203+
g.Expect(err).NotTo(gomega.HaveOccurred())
204+
if len(fetchedBackup.Finalizers) > 0 {
205+
fetchedBackup.SetFinalizers([]string{})
206+
g.Expect(factory.controllerRuntimeClient.Update(context.Background(), fetchedBackup)).
207+
NotTo(gomega.HaveOccurred())
208+
}
209+
210+
g.Expect(fetchedBackup.Finalizers).To(gomega.BeEmpty())
211+
}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
212+
}
213+
}
214+
}).WithTimeout(2 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
185215

186216
factory.Delete(&corev1.Namespace{
187217
ObjectMeta: metav1.ObjectMeta{
@@ -201,7 +231,7 @@ func (factory *Factory) checkIfNamespaceIsTerminating(name string) error {
201231
controllerClient := factory.GetControllerRuntimeClient()
202232

203233
namespace := &corev1.Namespace{}
204-
err := controllerClient.Get(ctx.Background(), client.ObjectKey{Name: name}, namespace)
234+
err := controllerClient.Get(context.Background(), client.ObjectKey{Name: name}, namespace)
205235
if err != nil {
206236
if k8serrors.IsNotFound(err) {
207237
return nil
@@ -229,7 +259,7 @@ func (factory *Factory) checkIfNamespaceIsTerminating(name string) error {
229259
deletionTimestamp.String(),
230260
)
231261
podList := &corev1.PodList{}
232-
err = controllerClient.List(ctx.Background(), podList, client.InNamespace(name))
262+
err = controllerClient.List(context.Background(), podList, client.InNamespace(name))
233263
if err != nil {
234264
return err
235265
}

‎e2e/fixtures/pods.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string)
132132
}
133133

134134
controllerClient := factory.GetControllerRuntimeClient()
135-
gomega.Eventually(func(g gomega.Gomega) bool {
135+
gomega.Eventually(func(g gomega.Gomega) {
136136
fetchedPod := &corev1.Pod{}
137137
g.Expect(controllerClient.Get(context.Background(), client.ObjectKeyFromObject(pod), fetchedPod)).
138138
NotTo(gomega.HaveOccurred())
@@ -144,9 +144,7 @@ func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string)
144144
}
145145

146146
g.Expect(fetchedPod.Finalizers).To(gomega.ConsistOf(finalizers))
147-
148-
return true
149-
}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.BeTrue())
147+
}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
150148
}
151149

152150
// GetProcessClass returns the Process class of this Pod.

‎e2e/scripts/remove_namespaces‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ do
2121
fi
2222

2323
echo "start deleting namespace ${ns}"
24+
for backup in $(kubectl -n ${ns} get fdbbackup --no-headers -o name);
25+
do
26+
echo "Removing finalizer from ${backup} in namespace ${ns}"
27+
kubectl -n ${ns} patch ${backup} -p '{"metadata":{"finalizers":null}}' --type=merge
28+
done
2429

2530
kubectl delete ns --ignore-not-found "${ns}"
2631
) &

0 commit comments

Comments
 (0)