diff --git a/cmd/testing-workload/main.go b/cmd/testing-workload/main.go index 13abc910264..50dc2eda26c 100644 --- a/cmd/testing-workload/main.go +++ b/cmd/testing-workload/main.go @@ -44,6 +44,7 @@ var ( longTxnSleepSeconds int maxLifeTimeSeconds int tiflashReplicas int + maxExecutionTime int // Flags for import action batchSize int @@ -72,6 +73,7 @@ const ( defaultSleepInterval = 100 defaultLongTxnSleepSeconds = 10 defaultMaxLifeTimeSeconds = 60 + defaultMaxExecutionTime = 2000 defaultBatchSize = 1000 defaultTotalRows = 500000 @@ -184,6 +186,7 @@ func parseFlag() { flag.IntVar(&longTxnSleepSeconds, "long-txn-sleep", defaultLongTxnSleepSeconds, "how many seconds to sleep to simulate a long transaction") flag.IntVar(&maxLifeTimeSeconds, "max-lifetime", defaultMaxLifeTimeSeconds, "max lifetime in seconds") flag.IntVar(&tiflashReplicas, "tiflash-replicas", 0, "replicas of tiflash") + flag.IntVar(&maxExecutionTime, "max-execution-time", defaultMaxExecutionTime, "max_execution_time of tidb") // Flags for import action flag.IntVar(&batchSize, "batch-size", defaultBatchSize, "batch size for import action") diff --git a/cmd/testing-workload/workload.go b/cmd/testing-workload/workload.go index 533a2c4a38b..a62fa51c241 100644 --- a/cmd/testing-workload/workload.go +++ b/cmd/testing-workload/workload.go @@ -38,13 +38,13 @@ func Workload(ctx context.Context, db *sql.DB) error { db.SetMaxOpenConns(maxConnections) // Set these variable to avoid too long retry time in testing. // Downtime may be short but default timeout is too long. - fmt.Println("try to set max_execution_time to 2000ms") - if _, err := db.Exec("set global max_execution_time = 2000;"); err != nil { - return fmt.Errorf("set max_execute_time failed: %w", err) + fmt.Printf("try to set max_execution_time to %dms\n", maxExecutionTime) + if _, err := db.Exec(fmt.Sprintf("set global max_execution_time = %d;", maxExecutionTime)); err != nil { + return fmt.Errorf("set max_execution_time failed: %w", err) } fmt.Println("try to set tidb_backoff_weight to 1") if _, err := db.Exec("set global tidb_backoff_weight = 1;"); err != nil { - return fmt.Errorf("set max_execute_time failed: %w", err) + return fmt.Errorf("set tidb_backoff_weight failed: %w", err) } table := "test.e2e_test" diff --git a/tests/e2e/data/tiflash.go b/tests/e2e/data/tiflash.go index 72f321ffb1c..a6e1cb60b76 100644 --- a/tests/e2e/data/tiflash.go +++ b/tests/e2e/data/tiflash.go @@ -76,7 +76,7 @@ s3-region = "local" api_version = 2 [flash] -graceful_wait_shutdown_timeout = 300 +graceful_wait_shutdown_timeout = 200 [storage.s3] endpoint = "http://minio:9000" @@ -88,7 +88,7 @@ secret_access_key = "test12345678" obj.Spec.Template.Spec.Overlay = &v1alpha1.Overlay{ Pod: &v1alpha1.PodOverlay{ Spec: &corev1.PodSpec{ - TerminationGracePeriodSeconds: ptr.To[int64](300), + TerminationGracePeriodSeconds: ptr.To[int64](250), }, }, } diff --git a/tests/e2e/framework/workload.go b/tests/e2e/framework/workload.go index e66d54b159b..e98b31fc514 100644 --- a/tests/e2e/framework/workload.go +++ b/tests/e2e/framework/workload.go @@ -176,6 +176,7 @@ func (w *Workload) MustRunWorkload(ctx context.Context, host string, opts ...wor // an arbitrary timeout // NOTE: maybe changed to use a http api to stop "--max-connections", "30", + "--max-execution-time", strconv.Itoa(o.MaxExecutionTime), "--tiflash-replicas", strconv.Itoa(o.TiFlashReplicas), } diff --git a/tests/e2e/framework/workload/options.go b/tests/e2e/framework/workload/options.go index b7b2cd05b83..d00cb87c3d9 100644 --- a/tests/e2e/framework/workload/options.go +++ b/tests/e2e/framework/workload/options.go @@ -29,6 +29,10 @@ type Options struct { RegionCount int + // MaxExecutionTime is max_execution_time of tidb in millisecond + // Default is 2000 + MaxExecutionTime int + TiFlashReplicas int } @@ -69,6 +73,12 @@ func RegionCount(count int) Option { }) } +func MaxExecutionTime(ms int) Option { + return WithOption(func(opts *Options) { + opts.MaxExecutionTime = ms + }) +} + func TiFlashReplicas(replicas int) Option { return WithOption(func(opts *Options) { opts.TiFlashReplicas = replicas @@ -80,6 +90,8 @@ func DefaultOptions() *Options { Port: 4000, User: "root", + MaxExecutionTime: 2000, + RegionCount: 500, } } diff --git a/tests/e2e/suite/availability/tiflash.go b/tests/e2e/suite/availability/tiflash.go index fc9f6393be9..87e7d53e6b8 100644 --- a/tests/e2e/suite/availability/tiflash.go +++ b/tests/e2e/suite/availability/tiflash.go @@ -82,7 +82,7 @@ var _ = ginkgo.Describe("TiFlash Availability Test", label.TiFlash, label.Update ginkgo.Context("NextGen", label.KindNextGen, label.P0, func() { workload := f.SetupWorkload() // TODO: still not work - ginkgo.PIt("No error when rolling update tiflash in next-gen", func(ctx context.Context) { + ginkgo.It("No error when rolling update tiflash in next-gen", func(ctx context.Context) { f.MustCreateS3(ctx) pdg := f.MustCreatePD(ctx, data.WithPDNextGen()) kvg := f.MustCreateTiKV(ctx, data.WithTiKVNextGen()) @@ -125,7 +125,13 @@ var _ = ginkgo.Describe("TiFlash Availability Test", label.TiFlash, label.Update f.Must(waiter.WaitPodsRollingUpdateOnce(nctx, f.Client, runtime.FromTiFlashGroup(fgw), 2, 0, waiter.LongTaskTimeout)) }() - done := workload.MustRunWorkload(nctx, data.DefaultTiDBServiceName, wopt.TiFlashReplicas(2)) + done := workload.MustRunWorkload(nctx, + data.DefaultTiDBServiceName, + wopt.TiFlashReplicas(2), + // Set max_execution_time to 4s for next-gen tiflash + // TODO: dig why 2000ms is not enough + wopt.MaxExecutionTime(4000), + ) patch := client.MergeFrom(fgc.DeepCopy()) fgc.Spec.Template.Labels = map[string]string{"test": "test"}