Skip to content

feat: support independent kvengine remote worker and compactor worker #6787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ti-chi-bot merged 3 commits into from
Apr 2, 2026
Merged

feat: support independent kvengine remote worker and compactor worker #6787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b4c01c
feat: support independent kvengine remote worker and compactor worker
liubog2008 Mar 31, 2026
b4ce28e
fix build
liubog2008 Mar 31, 2026
8e7d7eb
fix ut
liubog2008 Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion api/core/v1alpha1/tikv_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,15 +210,19 @@ type TiKVPreStop struct {

type (
CoprocessorReference corev1.LocalObjectReference
CompactorReference corev1.LocalObjectReference
WorkerReference corev1.LocalObjectReference
)

type TiKVRemoteWorkers struct {
// Default worker reference, if remote workers are enabled, this default worker ref must be set
Worker WorkerReference `json:"worker"`

// Coprocessor worker reference, if it's not set, use worker ref by default
// Coprocessor worker reference, if it's not set, use worker ref by default
Coprocessor *CoprocessorReference `json:"coprocessor,omitempty"`

// Compactor worker reference, if it's not set, use worker ref by default
Compactor *CompactorReference `json:"compactor,omitempty"`
}

type TiKVServer struct {
Expand Down
21 changes: 21 additions & 0 deletions api/core/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion api/meta/v1alpha1/feature.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package v1alpha1
// NOTE(liubo02): +enum is not supported now, we have to add all enum into comments
// NOTE(liubo02): It's supported by https://github.com/kubernetes-sigs/controller-tools/pull/1179
//
// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI;UsePDReadyAPIV2;UseTiFlashReadyAPI;MultiPDGroup;TiCDCDynamicSecretSyncer
// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI;UsePDReadyAPIV2;UseTiFlashReadyAPI;MultiPDGroup;TiCDCDynamicSecretSyncer;IndependentKVEngineWorker
// +enum
type Feature string

Expand Down Expand Up @@ -114,4 +114,9 @@ const (
// If this feature is enabled, TiCDC pods can dynamically load secrets with specific labels into pods
TiCDCDynamicSecretSyncer Feature = "TiCDCDynamicSecretSyncer"
TiCDCDynamicSecretSyncerStage FeatureStage = FeatureStageAlpha

// By default, kvengine.remote-worker-addr follows the coprocessor ref.
// If this feature is enabled, kvengine.remote-worker-addr will use the default worker ref.
IndependentKVEngineWorker Feature = "IndependentKVEngineWorker"
IndependentKVEngineWorkerStage FeatureStage = FeatureStageAlpha
)
2 changes: 2 additions & 0 deletions manifests/crd/core.pingcap.com_clusters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
required:
- name
Expand Down Expand Up @@ -369,6 +370,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
stage:
type: string
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_pdgroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_pds.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_resourcemanagergroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_resourcemanagers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_routergroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_routers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_schedulergroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_schedulers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_schedulinggroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_schedulings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_ticdcgroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_ticdcs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tidbgroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tidbs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tiflashes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tiflashgroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
18 changes: 17 additions & 1 deletion manifests/crd/core.pingcap.com_tikvgroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down Expand Up @@ -8623,8 +8624,23 @@ spec:
RemoteWorkers defines remote workers used by this tikv
It only works for nextgen
properties:
compactor:
description: Compactor worker reference, if it's not set,
use worker ref by default
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
type: object
x-kubernetes-map-type: atomic
coprocessor:
description: Coprocessor worker reference, if it's not
description: Coprocessor worker reference, if it's not
set, use worker ref by default
properties:
name:
Expand Down
18 changes: 17 additions & 1 deletion manifests/crd/core.pingcap.com_tikvs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down Expand Up @@ -8349,8 +8350,23 @@ spec:
RemoteWorkers defines remote workers used by this tikv
It only works for nextgen
properties:
compactor:
description: Compactor worker reference, if it's not set, use
worker ref by default
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
type: object
x-kubernetes-map-type: atomic
coprocessor:
description: Coprocessor worker reference, if it's not set, use
description: Coprocessor worker reference, if it's not set, use
worker ref by default
properties:
name:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tikvworkergroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tikvworkers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tiproxies.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tiproxygroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tsogroups.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
minReadySeconds:
Expand Down
1 change: 1 addition & 0 deletions manifests/crd/core.pingcap.com_tsos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ spec:
- UseTiFlashReadyAPI
- MultiPDGroup
- TiCDCDynamicSecretSyncer
- IndependentKVEngineWorker
type: string
type: array
image:
Expand Down
17 changes: 15 additions & 2 deletions pkg/configs/tikv/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
corev1 "k8s.io/api/core/v1"

"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
metav1alpha1 "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
coreutil "github.com/pingcap/tidb-operator/v2/pkg/apiutil/core/v1alpha1"
"github.com/pingcap/tidb-operator/v2/pkg/features"
"github.com/pingcap/tidb-operator/v2/pkg/runtime/scope"
)

Expand Down Expand Up @@ -68,7 +70,7 @@ type KVEngine struct {
RemoteWorkerAddr string `toml:"remote-worker-addr"`
}

func (c *Config) Overlay(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV) error {
func (c *Config) Overlay(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV, fg features.Gates) error {
if err := c.Validate(); err != nil {
return err
}
Expand Down Expand Up @@ -107,14 +109,25 @@ func (c *Config) Overlay(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV) error {
if c.DFS == nil {
c.DFS = &DFS{}
}

compactor := tikv.Spec.RemoteWorkers.Worker.Name
copr := tikv.Spec.RemoteWorkers.Worker.Name
worker := tikv.Spec.RemoteWorkers.Worker.Name

if tikv.Spec.RemoteWorkers.Compactor != nil {
compactor = tikv.Spec.RemoteWorkers.Compactor.Name
}
if tikv.Spec.RemoteWorkers.Coprocessor != nil {
copr = tikv.Spec.RemoteWorkers.Coprocessor.Name
}
if !fg.Enabled(metav1alpha1.IndependentKVEngineWorker) {
if tikv.Spec.RemoteWorkers.Coprocessor != nil {
worker = tikv.Spec.RemoteWorkers.Coprocessor.Name
}
}

c.KVEngine.RemoteCoprocessorAddr = coreutil.TiKVWorkerCoprocessorURLFromRef(cluster, copr)
c.KVEngine.RemoteWorkerAddr = coreutil.TiKVWorkerCoprocessorURLFromRef(cluster, copr)
c.KVEngine.RemoteWorkerAddr = coreutil.TiKVWorkerCoprocessorURLFromRef(cluster, worker)
c.DFS.RemoteCompactorAddr = coreutil.TiKVWorkerCompactorURLFromRef(cluster, compactor)
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/configs/tikv/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
"github.com/pingcap/tidb-operator/v2/pkg/features"
)

func TestValidate(t *testing.T) {
Expand Down Expand Up @@ -107,7 +108,7 @@ level = "info"`),
}

cfg := &Config{}
err := cfg.Overlay(cluster, tikv)
err := cfg.Overlay(cluster, tikv, features.NewFromFeatures(nil))
require.NoError(t, err)
assert.Equal(t, "[::]:20160", cfg.Server.Addr)
assert.Equal(t, "basic-tikv-0.basic-tikv-peer.ns1:20160", cfg.Server.AdvertiseAddr)
Expand Down
Loading
Loading