Skip to content

release-26.4: backup: TestBackupManifestFileCount failed #175764

Description

@cockroach-teamcity

backup.TestBackupManifestFileCount failed on release-26.4 @ 98b90b8131eb679e10a0ca4fbb6eb402252a1313:

Data race:

WARNING: DATA RACE
Write at 0x00c016b02270 by goroutine 2321:
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).canAttempt1PCEvaluation()
      pkg/kv/kvserver/replica_write.go:405 +0x69c
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch()
      pkg/kv/kvserver/replica_write.go:448 +0x19b
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal()
      pkg/kv/kvserver/replica_proposal.go:1096 +0x25a
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal()
      pkg/kv/kvserver/replica_proposal.go:1204 +0x14c
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose()
      pkg/kv/kvserver/replica_raft.go:129 +0x1da
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch()
      pkg/kv/kvserver/replica_write.go:190 +0xbb9
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries()
      pkg/kv/kvserver/replica_send.go:573 +0x682
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWorkStats()
      pkg/kv/kvserver/replica_send.go:224 +0xb44
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWorkStats()
      pkg/kv/kvserver/store_send.go:242 +0xc64
  github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWorkStats()
      pkg/kv/kvserver/stores.go:195 +0xfb
  github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal()
      pkg/server/node.go:1750 +0xbc9
  github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch()
      pkg/server/node.go:1943 +0x3f5
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1()
      pkg/rpc/context.go:822 +0x74
  github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.ServerInterceptor.func13()
      pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:60 +0x74b
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3()
      pkg/rpc/context.go:197 +0xf0
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor()
      pkg/rpc/auth.go:107 +0x335
  github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor-fm()
      <autogenerated>:1 +0xf0
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/rpc.gatewayRequestRecoveryInterceptor()
      pkg/rpc/metrics.go:826 +0x182
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/server.newGRPCServer.NewRequestMetricsInterceptor.func4()
      pkg/rpc/metrics.go:610 +0x3a1
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1()
      pkg/rpc/context.go:154 +0x6a
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr()
      pkg/util/stop/stopper.go:358 +0x236
  github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1()
      pkg/rpc/context.go:152 +0x14b
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUnaryServerInterceptors.bindUnaryServerInterceptorToHandler.func5()
      pkg/rpc/context.go:933 +0x88
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2()
      pkg/rpc/context.go:832 +0x75
  github.com/cockroachdb/cockroach/pkg/rpc.NewContext.ClientInterceptor.func13()
      pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:185 +0x4fc
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.getChainUnaryInvoker.func4()
      pkg/rpc/context.go:1017 +0x18c
  github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3()
      pkg/rpc/context.go:902 +0x470
  github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch()
      pkg/rpc/context.go:1025 +0xd0
  github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch()
      <autogenerated>:1 +0x1a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch()
      pkg/kv/kvclient/kvcoord/transport.go:245 +0x458
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext()
      pkg/kv/kvclient/kvcoord/transport.go:194 +0x134
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.raceTransport.SendNext()
      pkg/kv/kvclient/kvcoord/transport_race.go:80 +0x181
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*raceTransport).SendNext()
      <autogenerated>:1 +0x5e
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas()
      pkg/kv/kvclient/kvcoord/dist_sender.go:2944 +0x2a5b
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch()
      pkg/kv/kvclient/kvcoord/dist_sender.go:2353 +0x924
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges()
      pkg/kv/kvclient/kvcoord/dist_sender.go:1899 +0xf92
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send()
      pkg/kv/kvclient/kvcoord/dist_sender.go:1403 +0x825
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:77 +0x258
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:42 +0x13d
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts()
      pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:228 +0x82
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:162 +0x1bc
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:190 +0x40e
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:340 +0x29c
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnWriteBuffer).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_write_buffer.go:298 +0x2cb
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:124 +0x463
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked()
      pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:267 +0x7a2
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send()
      pkg/kv/kvclient/kvcoord/txn_coord_sender.go:574 +0xd74
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender()
      pkg/kv/db.go:1277 +0x17d
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send()
      pkg/kv/txn.go:1471 +0x6c5
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit()
      pkg/kv/txn.go:946 +0x1e6
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit()
      pkg/kv/txn.go:956 +0xa4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal()
      pkg/sql/conn_executor_exec.go:2898 +0xba5
  github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).commitTxn()
      pkg/sql/internal.go:1497 +0x766
  github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).newInternalExecutorWithTxn.func1()
      pkg/sql/internal.go:1970 +0xeb
  github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn.func4()
      pkg/sql/internal.go:2122 +0x847
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec()
      pkg/kv/txn.go:1206 +0xad
  github.com/cockroachdb/cockroach/pkg/kv.runTxn()
      pkg/kv/db.go:1195 +0x6b
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl()
      pkg/kv/db.go:1154 +0x23e
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn()
      pkg/kv/db.go:1125 +0x4f
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn-fm()
      <autogenerated>:1 +0xe
  github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn()
      pkg/sql/internal.go:2092 +0x6d1
  github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn()
      pkg/sql/internal.go:2010 +0xd4
  github.com/cockroachdb/cockroach/pkg/sql.(*internalDBWithOverrides).Txn()
      pkg/sql/internal.go:2177 +0x124
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).claimJobs()
      pkg/jobs/adopt.go:126 +0x328
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func5()
      pkg/jobs/registry.go:1061 +0x133
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).withSession()
      pkg/jobs/registry.go:996 +0x322
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).withSession()
      pkg/jobs/registry.go:986 +0x8a
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.(*Registry).Start.func1.func12()
      pkg/jobs/registry.go:1012 +0x4c
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).withSession()
      pkg/jobs/registry.go:986 +0x8a
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.(*Registry).Start.func1.func12()
      pkg/jobs/registry.go:1012 +0x4c
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1209 +0x4e1
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10()
      pkg/jobs/registry.go:1208 +0x4c6

Previous read at 0x00c016b02270 by goroutine 963:
  github.com/cockroachdb/cockroach/pkg/util/hlc.(*Timestamp).Size()
      bazel-out/k8-fastbuild/bin/pkg/util/hlc/hlc_go_proto_/github.com/cockroachdb/cockroach/pkg/util/hlc/timestamp.pb.go:253 +0x145
  github.com/cockroachdb/cockroach/pkg/storage/enginepb.(*TxnMeta).Size()
      bazel-out/k8-fastbuild/bin/pkg/storage/enginepb/enginepb_go_proto_/github.com/cockroachdb/cockroach/pkg/storage/enginepb/mvcc3.pb.go:1969 +0xf7
  github.com/cockroachdb/cockroach/pkg/roachpb.(*Transaction).Size()
      bazel-out/k8-fastbuild/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/data.pb.go:4470 +0x4d
  github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*Header).Size()
      bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:28237 +0x28a
  github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*BatchRequest).Size()
      bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:28411 +0x37
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.readAll()
      pkg/kv/kvclient/kvcoord/transport_race.go:39 +0x34
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.startRacer.func1()
      pkg/kv/kvclient/kvcoord/transport_race.go:127 +0x157
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1()
      pkg/util/stop/stopper.go:447 +0x112
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.gowrap1()
      pkg/util/stop/stopper.go:448 +0x46

Goroutine 2321 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      pkg/util/stop/stopper.go:445 +0x246
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      pkg/util/stop/stopper.go:368 +0xce9
  github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start()
      pkg/jobs/registry.go:1178 +0xae5
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      pkg/server/server_sql.go:1744 +0x14a4
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      pkg/server/server_sql.go:1656 +0xbed
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      pkg/server/server.go:2367 +0x6bca
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      pkg/server/server.go:2197 +0x4dd9
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      pkg/server/server.go:1862 +0x248f
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      pkg/server/testserver.go:959 +0x144
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:19 +0x71
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      pkg/testutils/testcluster/testcluster.go:575 +0x627
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      pkg/testutils/testcluster/testcluster.go:359 +0x91
  github.com/cockroachdb/cockroach/pkg/backup/backuptestutils.StartBackupRestoreTestCluster()
      pkg/backup/backuptestutils/testutils.go:201 +0x37c
  github.com/cockroachdb/cockroach/pkg/backup.backupRestoreTestSetupWithParams()
      pkg/backup/utils_test.go:75 +0x1be
  github.com/cockroachdb/cockroach/pkg/backup.backupRestoreTestSetup()
      pkg/backup/utils_test.go:84 +0x8b
  github.com/cockroachdb/cockroach/pkg/backup.TestBackupManifestFileCount()
      pkg/backup/backup_test.go:602 +0x138
  testing.tRunner()
      GOROOT/src/testing/testing.go:2036 +0x21c
  testing.(*T).Run.gowrap1()
      GOROOT/src/testing/testing.go:2101 +0x38

Goroutine 963 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      pkg/util/stop/stopper.go:445 +0x246
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      pkg/util/stop/stopper.go:368 +0x125
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.startRacer()
      pkg/kv/kvclient/kvcoord/transport_race.go:111 +0xd2
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.GRPCTransportFactory()
      pkg/kv/kvclient/kvcoord/transport_race.go:98 +0x289b
  github.com/cockroachdb/cockroach/pkg/server.NewServer()
      pkg/server/server.go:539 +0x2abe
  github.com/cockroachdb/cockroach/pkg/server.testServerFactoryImpl.New()
      pkg/server/testserver.go:2698 +0x4ac
  github.com/cockroachdb/cockroach/pkg/server.(*testServerFactoryImpl).New()
      <autogenerated>:1 +0x56
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.NewServer()
      pkg/testutils/serverutils/test_server_shim.go:376 +0x36b
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).AddServer()
      pkg/testutils/testcluster/testcluster.go:933 +0xa87
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.NewTestCluster()
      pkg/testutils/testcluster/testcluster.go:528 +0x16ce
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      pkg/testutils/testcluster/testcluster.go:358 +0x74
  github.com/cockroachdb/cockroach/pkg/backup/backuptestutils.StartBackupRestoreTestCluster()
      pkg/backup/backuptestutils/testutils.go:201 +0x37c
  github.com/cockroachdb/cockroach/pkg/backup.backupRestoreTestSetupWithParams()
      pkg/backup/utils_test.go:75 +0x1be
  github.com/cockroachdb/cockroach/pkg/backup.backupRestoreTestSetup()
      pkg/backup/utils_test.go:84 +0x8b
  github.com/cockroachdb/cockroach/pkg/backup.TestBackupManifestFileCount()
      pkg/backup/backup_test.go:602 +0x138
  testing.tRunner()
      GOROOT/src/testing/testing.go:2036 +0x21c
  testing.(*T).Run.gowrap1()
      GOROOT/src/testing/testing.go:2101 +0x38
Log preceding data race

=== RUN   TestBackupManifestFileCount
    test_log_scope.go:172: test logs captured to: outputs.zip/logTestBackupManifestFileCount2937661950
    test_log_scope.go:82: use -show-logs to present logs inline
    test_server_shim.go:168: automatically injected an external process virtual cluster under test; see comment at top of test_server_shim.go for details.
    test_server_shim.go:637: DRPC is enabled (via testserver factory defaults)

Parameters:

  • attempt=1
  • race=true
  • run=1
  • shard=32
Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachlabs/disaster-recovery

Improve this report!

Jira issue: CRDB-68720

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    A-disaster-recoveryC-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.T-disaster-recoverybranch-release-26.4Used to mark GA and release blockers, technical advisories, and bugs for 26.4release-blockerIndicates a release-blocker. Use with branch-release-2x.x label to denote which branch is blocked.

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions