Skip to content

Data race in TestDelaySyncTimestamp #9874

New issue
New issue
Open
#9899
Open
Data race in TestDelaySyncTimestamp#9874
#9899
Labels
type/ciThe issue is related to CI.
@rleungx

Description

@rleungx
rleungx
opened on Oct 27, 2025

Flaky Test

Which jobs are failing

==================
WARNING: DATA RACE
Write at 0x00c000886a18 by goroutine 14113:
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startCluster()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:502 +0xc56
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startCluster-fm()
      <autogenerated>:1 +0x47
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).campaignPrimary()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:305 +0xd0b
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).primaryElectionLoop()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:272 +0xc1a
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServerLoop.gowrap1()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:171 +0x33

Previous read at 0x00c000886a18 by goroutine 14114:
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).updatePDMemberLoop()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:223 +0x129a
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServerLoop.gowrap2()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:172 +0x33

Goroutine 14113 (running) created at:
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServerLoop()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:171 +0x21c
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServer()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:478 +0x1286
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).Run()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:165 +0x1bc
  github.com/tikv/pd/tests.NewSchedulingTestServer()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:204 +0x99
  github.com/tikv/pd/tests.(*TestSchedulingCluster).AddServer()
      /home/prow/go/src/github.com/tikv/pd/tests/scheduling_cluster.go:77 +0x512
  github.com/tikv/pd/tests.NewTestSchedulingCluster()
      /home/prow/go/src/github.com/tikv/pd/tests/scheduling_cluster.go:55 +0x2c8
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).startCluster()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:624 +0xc3a
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:590 +0x239
  github.com/pingcap/failpoint.parseTerm()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:149 +0x233
  github.com/pingcap/failpoint.parse()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:126 +0xa5
  github.com/pingcap/failpoint.newTerms()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:98 +0x3e
  github.com/pingcap/failpoint.(*Failpoint).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoint.go:58 +0x3e
  github.com/pingcap/failpoint.(*Failpoints).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:105 +0x296
  github.com/pingcap/failpoint.Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:255 +0x13a
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:584 +0x13b
  github.com/pingcap/failpoint.parseTerm()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:149 +0x233
  github.com/pingcap/failpoint.parse()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:126 +0xa5
  github.com/pingcap/failpoint.newTerms()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:98 +0x3e
  github.com/pingcap/failpoint.(*Failpoint).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoint.go:58 +0x3e
  github.com/pingcap/failpoint.(*Failpoints).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:105 +0x296
  github.com/pingcap/failpoint.Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:255 +0xfc
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:583 +0xfd
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).RunFunc()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:398 +0xf6
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).Reset()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:561 +0xf6
  github.com/tikv/pd/tests/server/tso_test.(*tsoTestSuite).TearDownTest()
      /home/prow/go/src/github.com/tikv/pd/tests/server/tso/tso_test.go:64 +0x25
  github.com/stretchr/testify/suite.Run.func1.1()
      /home/prow/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:184 +0x335
  runtime.deferreturn()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:611 +0x5d
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInNonMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:578 +0x12d
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).RunTestInNonMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:405 +0xf3
  github.com/tikv/pd/tests/server/tso_test.(*tsoTestSuite).TestDelaySyncTimestamp()
      /home/prow/go/src/github.com/tikv/pd/tests/server/tso/tso_test.go:112 +0x6f
  runtime.call16()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:775 +0x42
  reflect.Value.Call()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/reflect/value.go:368 +0xb5
  github.com/stretchr/testify/suite.Run.func1()
      /home/prow/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:202 +0x6f1
  testing.tRunner()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/testing/testing.go:1690 +0x226
  testing.(*T).Run.gowrap1()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/testing/testing.go:1743 +0x44

Goroutine 14114 (running) created at:
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServerLoop()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:172 +0x285
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).startServer()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:478 +0x1286
  github.com/tikv/pd/pkg/mcs/scheduling/server.(*Server).Run()
      /home/prow/go/src/github.com/tikv/pd/pkg/mcs/scheduling/server/server.go:165 +0x1bc
  github.com/tikv/pd/tests.NewSchedulingTestServer()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:204 +0x99
  github.com/tikv/pd/tests.(*TestSchedulingCluster).AddServer()
      /home/prow/go/src/github.com/tikv/pd/tests/scheduling_cluster.go:77 +0x512
  github.com/tikv/pd/tests.NewTestSchedulingCluster()
      /home/prow/go/src/github.com/tikv/pd/tests/scheduling_cluster.go:55 +0x2c8
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).startCluster()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:624 +0xc3a
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:590 +0x239
  github.com/pingcap/failpoint.parseTerm()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:149 +0x233
  github.com/pingcap/failpoint.parse()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:126 +0xa5
  github.com/pingcap/failpoint.newTerms()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:98 +0x3e
  github.com/pingcap/failpoint.(*Failpoint).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoint.go:58 +0x3e
  github.com/pingcap/failpoint.(*Failpoints).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:105 +0x296
  github.com/pingcap/failpoint.Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:255 +0x13a
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:584 +0x13b
  github.com/pingcap/failpoint.parseTerm()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:149 +0x233
  github.com/pingcap/failpoint.parse()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:126 +0xa5
  github.com/pingcap/failpoint.newTerms()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:98 +0x3e
  github.com/pingcap/failpoint.(*Failpoint).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoint.go:58 +0x3e
  github.com/pingcap/failpoint.(*Failpoints).Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:105 +0x296
  github.com/pingcap/failpoint.Enable()
      /home/prow/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:255 +0xfc
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:583 +0xfd
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).RunFunc()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:398 +0xf6
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).Reset()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:561 +0xf6
  github.com/tikv/pd/tests/server/tso_test.(*tsoTestSuite).TearDownTest()
      /home/prow/go/src/github.com/tikv/pd/tests/server/tso/tso_test.go:64 +0x25
  github.com/stretchr/testify/suite.Run.func1.1()
      /home/prow/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:184 +0x335
  runtime.deferreturn()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:611 +0x5d
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).runFuncInNonMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:578 +0x12d
  github.com/tikv/pd/tests.(*SchedulingTestEnvironment).RunTestInNonMicroserviceEnv()
      /home/prow/go/src/github.com/tikv/pd/tests/testutil.go:405 +0xf3
  github.com/tikv/pd/tests/server/tso_test.(*tsoTestSuite).TestDelaySyncTimestamp()
      /home/prow/go/src/github.com/tikv/pd/tests/server/tso/tso_test.go:112 +0x6f
  runtime.call16()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:775 +0x42
  reflect.Value.Call()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/reflect/value.go:368 +0xb5
  github.com/stretchr/testify/suite.Run.func1()
      /home/prow/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:202 +0x6f1
  testing.tRunner()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/testing/testing.go:1690 +0x226
  testing.(*T).Run.gowrap1()
      /home/prow/go/pkg/mod/golang.org/[email protected]/src/testing/testing.go:1743 +0x44
==================

CI link

https://prow.tidb.net/view/gs/prow-tidb-logs/pr-logs/pull/tikv_pd/9873/pull-unit-test-next-gen/1982714897699442688

Reason for failure (if possible)

Anything else

Metadata

Metadata

Assignees

No one assigned

    Labels

    type/ciThe issue is related to CI.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions