Skip to content

Commit c7112ac

Browse files
henrybear327henrybear327
committed
Add new target for issue20221 and cleanup changes
Signed-off-by: Chun-Hung Tseng <[email protected]>
1 parent af9651e commit c7112ac

File tree

4 files changed

+85
-54
lines changed

4 files changed

+85
-54
lines changed

‎tests/robustness/Makefile‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ test-robustness-issue19179: /tmp/etcd-v3.5.17-failpoints/bin
7575
GO_TEST_FLAGS='-v -run=TestRobustnessRegression/Issue19179 -count 200 -failfast --bin-dir=/tmp/etcd-v3.5.17-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \
7676
echo "Failed to reproduce" || echo "Successful reproduction"
7777

78+
.PHONY: test-robustness-issue20221
79+
test-robustness-issue20221: /tmp/etcd-v3.6.4-failpoints/bin
80+
GO_TEST_FLAGS='-v -run=TestRobustnessRegression/issue20221 -count 200 -failfast --bin-dir=/tmp/etcd-v3.6.4-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \
81+
echo "Failed to reproduce" || echo "Successful reproduction"
82+
7883
# Etcd API usage by Kubernetes
7984

8085
.PHONY: k8s-coverage
@@ -120,14 +125,6 @@ $(GOPATH)/bin/gofail: $(REPOSITORY_ROOT)/tools/mod/go.mod $(REPOSITORY_ROOT)/too
120125
$(MAKE) gofail-enable; \
121126
$(MAKE) build;
122127

123-
/tmp/etcd-v3.6.0-failpoints/bin: $(GOPATH)/bin/gofail
124-
rm -rf /tmp/etcd-v3.6.0-failpoints/
125-
mkdir -p /tmp/etcd-v3.6.0-failpoints/
126-
cd /tmp/etcd-v3.6.0-failpoints/; \
127-
git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \
128-
$(MAKE) gofail-enable; \
129-
$(MAKE) build;
130-
131128
/tmp/etcd-release-3.6-failpoints/bin: $(GOPATH)/bin/gofail
132129
rm -rf /tmp/etcd-release-3.6-failpoints/
133130
mkdir -p /tmp/etcd-release-3.6-failpoints/
@@ -136,6 +133,19 @@ $(GOPATH)/bin/gofail: $(REPOSITORY_ROOT)/tools/mod/go.mod $(REPOSITORY_ROOT)/too
136133
$(MAKE) gofail-enable; \
137134
$(MAKE) build;
138135

136+
/tmp/etcd-v3.6.0-failpoints/bin:
137+
/tmp/etcd-v3.6.%-failpoints/bin: $(GOPATH)/bin/gofail
138+
rm -rf /tmp/etcd-v3.6.$*-failpoints/
139+
mkdir -p /tmp/etcd-v3.6.$*-failpoints/
140+
cd /tmp/etcd-v3.6.$*-failpoints/; \
141+
git clone --depth 1 --branch v3.6.$* https://github.com/etcd-io/etcd.git .; \
142+
go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
143+
(cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
144+
(cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
145+
(cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
146+
(cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
147+
FAILPOINTS=true ./build;
148+
139149
/tmp/etcd-v3.5.2-failpoints/bin:
140150
/tmp/etcd-v3.5.4-failpoints/bin:
141151
/tmp/etcd-v3.5.5-failpoints/bin:

‎tests/robustness/client/watch.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,8 @@ func openWatchPeriodically(ctx context.Context, g *errgroup.Group, c *RecordingC
154154
return err
155155
}
156156
// rev := resp.Header.Revision + (rand.Int64N(20) - 10) // reproduce OK (<2 min on my machine)
157-
rev := resp.Header.Revision - 10 //reproduce OK (<1 min on my machine)
157+
rev := resp.Header.Revision - 10 // reproduce OK (<1 min on my machine)
158+
// rev := resp.Header.Revision // reproduce TBD
158159
// rev := int64(0) // no reproduction
159160

160161
watchCtx, cancel := context.WithCancel(ctx)

‎tests/robustness/failpoint/failpoint.go‎

Lines changed: 17 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,27 +36,23 @@ const (
3636
)
3737

3838
var allFailpoints = []Failpoint{
39-
// KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
40-
// DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
41-
// BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
42-
// BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
43-
// CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
44-
// CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
45-
// RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
46-
// RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
47-
// BeforeApplyOneConfChangeSleep,
48-
// MemberReplace,
49-
// MemberDowngrade,
50-
// MemberDowngradeUpgrade,
51-
// DropPeerNetwork,
52-
// RaftBeforeSaveSleep,
53-
// RaftAfterSaveSleep,
54-
// ApplyBeforeOpenSnapshot,
55-
// SleepBeforeSendWatchResponse,
56-
RaftAfterSaveSnapPanic,
57-
RaftBeforeApplySnapPanic,
58-
RaftAfterApplySnapPanic,
59-
RaftAfterWALReleasePanic,
39+
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
40+
DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
41+
BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
42+
BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
43+
CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
44+
CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
45+
RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
46+
RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
47+
BeforeApplyOneConfChangeSleep,
48+
MemberReplace,
49+
MemberDowngrade,
50+
MemberDowngradeUpgrade,
51+
DropPeerNetwork,
52+
RaftBeforeSaveSleep,
53+
RaftAfterSaveSleep,
54+
ApplyBeforeOpenSnapshot,
55+
SleepBeforeSendWatchResponse,
6056
}
6157

6258
func PickRandom(clus *e2e.EtcdProcessCluster, profile traffic.Profile) (Failpoint, error) {

‎tests/robustness/scenarios/scenarios.go‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424

2525
"go.etcd.io/etcd/api/v3/version"
2626
"go.etcd.io/etcd/client/pkg/v3/fileutil"
27+
"go.etcd.io/etcd/server/v3/etcdserver"
2728
"go.etcd.io/etcd/tests/v3/framework/e2e"
2829
"go.etcd.io/etcd/tests/v3/robustness/client"
2930
"go.etcd.io/etcd/tests/v3/robustness/failpoint"
@@ -39,16 +40,16 @@ type TrafficProfile struct {
3940
}
4041

4142
var trafficProfiles = []TrafficProfile{
42-
// {
43-
// Name: "EtcdHighTraffic",
44-
// Traffic: traffic.EtcdPut,
45-
// Profile: traffic.HighTrafficProfile,
46-
// },
47-
// {
48-
// Name: "EtcdTrafficDeleteLeases",
49-
// Traffic: traffic.EtcdPutDeleteLease,
50-
// Profile: traffic.LowTraffic,
51-
// },
43+
{
44+
Name: "EtcdHighTraffic",
45+
Traffic: traffic.EtcdPut,
46+
Profile: traffic.HighTrafficProfile,
47+
},
48+
{
49+
Name: "EtcdTrafficDeleteLeases",
50+
Traffic: traffic.EtcdPutDeleteLease,
51+
Profile: traffic.LowTraffic,
52+
},
5253
{
5354
Name: "KubernetesHighTraffic",
5455
Traffic: traffic.Kubernetes,
@@ -93,11 +94,11 @@ func Exploratory(_ *testing.T) []TestScenario {
9394
mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices))
9495

9596
baseOptions := []e2e.EPClusterOption{
96-
options.WithSnapshotCount(50),
97+
options.WithSnapshotCount(50, 100, 1000),
9798
options.WithSubsetOptions(randomizableOptions...),
9899
e2e.WithGoFailEnabled(true),
99100
// Set a low minimal compaction batch limit to allow for triggering multi batch compaction failpoints.
100-
options.WithCompactionBatchLimit(10),
101+
options.WithCompactionBatchLimit(10, 100, 1000),
101102
e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
102103
}
103104

@@ -106,20 +107,20 @@ func Exploratory(_ *testing.T) []TestScenario {
106107
}
107108

108109
if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
109-
baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100))
110+
baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries))
110111
}
111112
scenarios := []TestScenario{}
112-
// for _, tp := range trafficProfiles {
113-
// name := filepath.Join(tp.Name, "ClusterOfSize1")
114-
// clusterOfSize1Options := baseOptions
115-
// clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
116-
// scenarios = append(scenarios, TestScenario{
117-
// Name: name,
118-
// Traffic: tp.Traffic,
119-
// Profile: tp.Profile,
120-
// Cluster: *e2e.NewConfig(clusterOfSize1Options...),
121-
// })
122-
// }
113+
for _, tp := range trafficProfiles {
114+
name := filepath.Join(tp.Name, "ClusterOfSize1")
115+
clusterOfSize1Options := baseOptions
116+
clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
117+
scenarios = append(scenarios, TestScenario{
118+
Name: name,
119+
Traffic: tp.Traffic,
120+
Profile: tp.Profile,
121+
Cluster: *e2e.NewConfig(clusterOfSize1Options...),
122+
})
123+
}
123124

124125
for _, tp := range trafficProfiles {
125126
name := filepath.Join(tp.Name, "ClusterOfSize3")
@@ -260,6 +261,7 @@ func Regression(t *testing.T) []TestScenario {
260261
e2e.WithGoFailEnabled(true),
261262
),
262263
})
264+
263265
scenarios = append(scenarios, TestScenario{
264266
Name: "Issue18089",
265267
Profile: traffic.LowTraffic.WithCompactionPeriod(100 * time.Millisecond), // Use frequent compaction for high reproduce rate
@@ -270,6 +272,7 @@ func Regression(t *testing.T) []TestScenario {
270272
e2e.WithGoFailEnabled(true),
271273
),
272274
})
275+
273276
if v.Compare(version.V3_5) >= 0 {
274277
opts := []e2e.EPClusterOption{
275278
e2e.WithSnapshotCount(100),
@@ -279,6 +282,7 @@ func Regression(t *testing.T) []TestScenario {
279282
if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
280283
opts = append(opts, e2e.WithSnapshotCatchUpEntries(100))
281284
}
285+
282286
scenarios = append(scenarios, TestScenario{
283287
Name: "Issue15271",
284288
Failpoint: failpoint.BlackholeUntilSnapshot,
@@ -287,5 +291,25 @@ func Regression(t *testing.T) []TestScenario {
287291
Cluster: *e2e.NewConfig(opts...),
288292
})
289293
}
294+
295+
scenarios = append(scenarios, TestScenario{
296+
Name: "issue20221",
297+
Profile: traffic.HighTrafficProfile.WithoutCompaction(),
298+
Failpoint: failpoint.RaftAfterSaveSnapPanic,
299+
// Failpoint: failpoint.RaftBeforeApplySnapPanic,
300+
// Failpoint: failpoint.RaftAfterApplySnapPanic,
301+
// Failpoint: failpoint.RaftAfterWALReleasePanic,
302+
Traffic: traffic.Kubernetes,
303+
Cluster: *e2e.NewConfig(
304+
e2e.WithClusterSize(3),
305+
e2e.WithCompactionBatchLimit(10),
306+
e2e.WithSnapshotCount(50),
307+
e2e.WithSnapshotCatchUpEntries(100),
308+
e2e.WithGoFailEnabled(true),
309+
e2e.WithPeerProxy(true),
310+
e2e.WithIsPeerTLS(true),
311+
),
312+
})
313+
290314
return scenarios
291315
}

0 commit comments

Comments
 (0)