test(e2e): enable next-gen tiflash avail test

cmd/testing-workload/main.go

@@ -44,6 +44,7 @@ var (
 	longTxnSleepSeconds int
 	maxLifeTimeSeconds  int
 	tiflashReplicas     int
+	maxExecutionTime    int
 
 	// Flags for import action
 	batchSize        int
@@ -72,6 +73,7 @@ const (
 	defaultSleepInterval       = 100
 	defaultLongTxnSleepSeconds = 10
 	defaultMaxLifeTimeSeconds  = 60
+	defaultMaxExecutionTime    = 2000
 
 	defaultBatchSize = 1000
 	defaultTotalRows = 500000
@@ -184,6 +186,7 @@ func parseFlag() {
 	flag.IntVar(&longTxnSleepSeconds, "long-txn-sleep", defaultLongTxnSleepSeconds, "how many seconds to sleep to simulate a long transaction")
 	flag.IntVar(&maxLifeTimeSeconds, "max-lifetime", defaultMaxLifeTimeSeconds, "max lifetime in seconds")
 	flag.IntVar(&tiflashReplicas, "tiflash-replicas", 0, "replicas of tiflash")
+	flag.IntVar(&maxExecutionTime, "max-execution-time", defaultMaxExecutionTime, "max_execution_time of tidb")
 
 	// Flags for import action
 	flag.IntVar(&batchSize, "batch-size", defaultBatchSize, "batch size for import action")

cmd/testing-workload/workload.go

@@ -38,13 +38,13 @@ func Workload(ctx context.Context, db *sql.DB) error {
 	db.SetMaxOpenConns(maxConnections)
 	// Set these variable to avoid too long retry time in testing.
 	// Downtime may be short but default timeout is too long.
-	fmt.Println("try to set max_execution_time to 2000ms")
-	if _, err := db.Exec("set global max_execution_time = 2000;"); err != nil {
-		return fmt.Errorf("set max_execute_time failed: %w", err)
+	fmt.Printf("try to set max_execution_time to %dms\n", maxExecutionTime)
+	if _, err := db.Exec(fmt.Sprintf("set global max_execution_time = %d;", maxExecutionTime)); err != nil {
+		return fmt.Errorf("set max_execution_time failed: %w", err)
 	}
 	fmt.Println("try to set tidb_backoff_weight to 1")
 	if _, err := db.Exec("set global tidb_backoff_weight = 1;"); err != nil {
-		return fmt.Errorf("set max_execute_time failed: %w", err)
+		return fmt.Errorf("set tidb_backoff_weight failed: %w", err)
 	}
 
 	table := "test.e2e_test"

tests/e2e/data/tiflash.go

@@ -76,7 +76,7 @@ s3-region = "local"
 api_version = 2
 
 [flash]
-graceful_wait_shutdown_timeout = 300
+graceful_wait_shutdown_timeout = 200
 
 [storage.s3]
 endpoint = "http://minio:9000"
@@ -88,7 +88,7 @@ secret_access_key = "test12345678"
 		obj.Spec.Template.Spec.Overlay = &v1alpha1.Overlay{
 			Pod: &v1alpha1.PodOverlay{
 				Spec: &corev1.PodSpec{
-					TerminationGracePeriodSeconds: ptr.To[int64](300),
+					TerminationGracePeriodSeconds: ptr.To[int64](250),
 				},
 			},
 		}

tests/e2e/framework/workload.go

@@ -176,6 +176,7 @@ func (w *Workload) MustRunWorkload(ctx context.Context, host string, opts ...wor
 		// an arbitrary timeout
 		// NOTE: maybe changed to use a http api to stop
 		"--max-connections", "30",
+		"--max-execution-time", strconv.Itoa(o.MaxExecutionTime),
 		"--tiflash-replicas", strconv.Itoa(o.TiFlashReplicas),
 	}
 

tests/e2e/framework/workload/options.go

@@ -29,6 +29,10 @@ type Options struct {
 
 	RegionCount int
 
+	// MaxExecutionTime is max_execution_time of tidb in millisecond
+	// Default is 2000
+	MaxExecutionTime int
+
 	TiFlashReplicas int
 }
 
@@ -69,6 +73,12 @@ func RegionCount(count int) Option {
 	})
 }
 
+func MaxExecutionTime(ms int) Option {
+	return WithOption(func(opts *Options) {
+		opts.MaxExecutionTime = ms
+	})
+}
+
 func TiFlashReplicas(replicas int) Option {
 	return WithOption(func(opts *Options) {
 		opts.TiFlashReplicas = replicas
@@ -80,6 +90,8 @@ func DefaultOptions() *Options {
 		Port: 4000,
 		User: "root",
 
+		MaxExecutionTime: 2000,
+
 		RegionCount: 500,
 	}
 }

tests/e2e/suite/availability/tiflash.go

@@ -82,7 +82,7 @@ var _ = ginkgo.Describe("TiFlash Availability Test", label.TiFlash, label.Update
 	ginkgo.Context("NextGen", label.KindNextGen, label.P0, func() {
 		workload := f.SetupWorkload()
 		// TODO: still not work
-		ginkgo.PIt("No error when rolling update tiflash in next-gen", func(ctx context.Context) {
+		ginkgo.It("No error when rolling update tiflash in next-gen", func(ctx context.Context) {
 			f.MustCreateS3(ctx)
 			pdg := f.MustCreatePD(ctx, data.WithPDNextGen())
 			kvg := f.MustCreateTiKV(ctx, data.WithTiKVNextGen())
@@ -125,7 +125,13 @@ var _ = ginkgo.Describe("TiFlash Availability Test", label.TiFlash, label.Update
 				f.Must(waiter.WaitPodsRollingUpdateOnce(nctx, f.Client, runtime.FromTiFlashGroup(fgw), 2, 0, waiter.LongTaskTimeout))
 			}()
 
-			done := workload.MustRunWorkload(nctx, data.DefaultTiDBServiceName, wopt.TiFlashReplicas(2))
+			done := workload.MustRunWorkload(nctx,
+				data.DefaultTiDBServiceName,
+				wopt.TiFlashReplicas(2),
+				// Set max_execution_time to 4s for next-gen tiflash
+				// TODO: dig why 2000ms is not enough
+				wopt.MaxExecutionTime(4000),
+			)
 
 			patch := client.MergeFrom(fgc.DeepCopy())
 			fgc.Spec.Template.Labels = map[string]string{"test": "test"}