add a control switch for watch middleware

zhouhaoA1 · xyz2277 · OrangeBao · zhouhaoA1 · commit a2e7068678c4 · 2023-11-20T17:43:17.000+08:00
Co-authored-by: zhangyongxi &lt;zhangyongxi_yewu@cmss.chinamobile.com&gt;
Co-authored-by: baoyinghai &lt;baoyinghai_yewu@cmss.chinamobile.com&gt;
Co-authored-by: wuyingjun &lt;wuyingjun_yewu@cmss.chinamobile.com&gt;
Signed-off-by: zhouhaoA1 &lt;zhouhao_yewu@cmss.chinamobile.com&gt;
diff --git a/cmd/apiserver/app/options/options.go b/cmd/apiserver/app/options/options.go
@@ -26,6 +26,7 @@ import (
 	storageoptions "github.com/clusterpedia-io/clusterpedia/pkg/storage/options"
 	"github.com/clusterpedia-io/clusterpedia/pkg/watcher"
 	watchcomponents "github.com/clusterpedia-io/clusterpedia/pkg/watcher/components"
+	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/middleware"
 	watchoptions "github.com/clusterpedia-io/clusterpedia/pkg/watcher/options"
 )
 
@@ -46,7 +47,7 @@ type ClusterPediaServerOptions struct {
 
 	Storage *storageoptions.StorageOptions
 
-	Subscriber *watchoptions.MiddlerwareOptions
+	Subscriber *watchoptions.MiddlewareOptions
 }
 
 func NewServerOptions() *ClusterPediaServerOptions {
@@ -128,11 +129,13 @@ func (o *ClusterPediaServerOptions) Config() (*apiserver.Config, error) {
 		StorageFactory: storage,
 	}
 
-	err = watcher.NewSubscriber(o.Subscriber)
-	watchcomponents.InitEventCacheSize(o.Subscriber.CacheSize)
-
-	if err != nil {
-		return nil, err
+	middleware.SubscriberEnabled = o.Subscriber.Enabled
+	if middleware.SubscriberEnabled {
+		err = watcher.NewSubscriber(o.Subscriber)
+		if err != nil {
+			return nil, err
+		}
+		watchcomponents.InitEventCacheSize(o.Subscriber.CacheSize)
 	}
 
 	return config, nil
diff --git a/cmd/clustersynchro-manager/app/options/options.go b/cmd/clustersynchro-manager/app/options/options.go
@@ -27,6 +27,7 @@ import (
 	"github.com/clusterpedia-io/clusterpedia/pkg/storage"
 	storageoptions "github.com/clusterpedia-io/clusterpedia/pkg/storage/options"
 	"github.com/clusterpedia-io/clusterpedia/pkg/watcher"
+	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/middleware"
 	watchoptions "github.com/clusterpedia-io/clusterpedia/pkg/watcher/options"
 )
 
@@ -47,7 +48,7 @@ type Options struct {
 	KubeStateMetrics *kubestatemetrics.Options
 
 	WorkerNumber int // WorkerNumber is the number of worker goroutines
-	Publisher    *watchoptions.MiddlerwareOptions
+	Publisher    *watchoptions.MiddlewareOptions
 }
 
 func NewClusterSynchroManagerOptions() (*Options, error) {
@@ -130,9 +131,12 @@ func (o *Options) Config() (*config.Config, error) {
 		return nil, err
 	}
 
-	err = watcher.NewPulisher(o.Publisher)
-	if err != nil {
-		return nil, err
+	middleware.PublisherEnabled = o.Publisher.Enabled
+	if middleware.PublisherEnabled {
+		err = watcher.NewPulisher(o.Publisher)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	kubeconfig, err := clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig)
diff --git a/cmd/clustersynchro-manager/app/synchro.go b/cmd/clustersynchro-manager/app/synchro.go
@@ -26,6 +26,8 @@ import (
 	"github.com/clusterpedia-io/clusterpedia/pkg/synchromanager"
 	clusterpediafeature "github.com/clusterpedia-io/clusterpedia/pkg/utils/feature"
 	"github.com/clusterpedia-io/clusterpedia/pkg/version/verflag"
+	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/components"
+	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/middleware"
 )
 
 func init() {
@@ -98,6 +100,12 @@ func Run(ctx context.Context, c *config.Config) error {
 	}
 
 	if !c.LeaderElection.LeaderElect {
+		if middleware.PublisherEnabled {
+			err := middleware.GlobalPublisher.InitPublisher(ctx)
+			if err != nil {
+				return err
+			}
+		}
 		synchromanager.Run(c.WorkerNumber, ctx.Done())
 		return nil
 	}
@@ -138,13 +146,23 @@ func Run(ctx context.Context, c *config.Config) error {
 				defer close(done)
 
 				stopCh := ctx.Done()
+				if middleware.PublisherEnabled {
+					err := middleware.GlobalPublisher.InitPublisher(ctx)
+					if err != nil {
+						return
+					}
+				}
 				synchromanager.Run(c.WorkerNumber, stopCh)
 			},
 			OnStoppedLeading: func() {
 				klog.Info("leaderelection lost")
 				if done != nil {
 					<-done
 				}
+				if middleware.PublisherEnabled {
+					middleware.GlobalPublisher.StopPublisher()
+					components.EC.CloseChannels()
+				}
 			},
 		},
 	})
diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
@@ -110,10 +110,12 @@ func (config completedConfig) New() (*ClusterPediaServer, error) {
 
 	// init event cache pool
 	eventStop := make(chan struct{})
-	watchcomponents.InitEventCachePool(eventStop)
-	err := middleware.GlobalSubscriber.InitSubscriber(eventStop)
-	if err != nil {
-		return nil, err
+	if middleware.SubscriberEnabled {
+		watchcomponents.InitEventCachePool(eventStop)
+		err := middleware.GlobalSubscriber.InitSubscriber(eventStop)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config.ClientConfig)
diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go
diff --git a/pkg/storage/internalstorage/resource_storage_test.go b/pkg/storage/internalstorage/resource_storage_test.go
@@ -188,13 +188,13 @@ func TestResourceStorage_genGetObjectQuery(t *testing.T) {
 	}{
 		{
 			"empty",
-			schema.GroupVersionResource{},
+			schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
 			"",
 			"",
 			"",
 			expected{
-				`SELECT "object" FROM "resources" WHERE "cluster" = '' AND "group" = '' AND "name" = '' AND "namespace" = '' AND "resource" = '' AND "version" = '' ORDER BY "resources"."id" LIMIT 1`,
-				"SELECT `object` FROM `resources` WHERE `cluster` = '' AND `group` = '' AND `name` = '' AND `namespace` = '' AND `resource` = '' AND `version` = '' ORDER BY `resources`.`id` LIMIT 1",
+				`SELECT cluster_resource_version, object FROM "apps_v1_deployments" WHERE "deleted" = false AND "name" = '' AND "namespace" = '' ORDER BY "apps_v1_deployments"."id" LIMIT 1`,
+				"SELECT cluster_resource_version, object FROM `apps_v1_deployments` WHERE `deleted` = false AND `name` = '' AND `namespace` = '' ORDER BY `apps_v1_deployments`.`id` LIMIT 1",
 				"",
 			},
 		},
@@ -205,8 +205,8 @@ func TestResourceStorage_genGetObjectQuery(t *testing.T) {
 			"ns-1",
 			"resource-1",
 			expected{
-				`SELECT "object" FROM "resources" WHERE "cluster" = 'cluster-1' AND "group" = 'apps' AND "name" = 'resource-1' AND "namespace" = 'ns-1' AND "resource" = 'deployments' AND "version" = 'v1' ORDER BY "resources"."id" LIMIT 1`,
-				"SELECT `object` FROM `resources` WHERE `cluster` = 'cluster-1' AND `group` = 'apps' AND `name` = 'resource-1' AND `namespace` = 'ns-1' AND `resource` = 'deployments' AND `version` = 'v1' ORDER BY `resources`.`id` LIMIT 1",
+				`SELECT cluster_resource_version, object FROM "apps_v1_deployments" WHERE "cluster" = 'cluster-1' AND "deleted" = false AND "name" = 'resource-1' AND "namespace" = 'ns-1' ORDER BY "apps_v1_deployments"."id" LIMIT 1`,
+				"SELECT cluster_resource_version, object FROM `apps_v1_deployments` WHERE `cluster` = 'cluster-1' AND `deleted` = false AND `name` = 'resource-1' AND `namespace` = 'ns-1' ORDER BY `apps_v1_deployments`.`id` LIMIT 1",
 				"",
 			},
 		},
@@ -250,8 +250,8 @@ func TestResourceStorage_genListObjectQuery(t *testing.T) {
 			appsv1.SchemeGroupVersion.WithResource("deployments"),
 			&internal.ListOptions{},
 			expected{
-				`SELECT * FROM "resources" WHERE "group" = 'apps' AND "resource" = 'deployments' AND "version" = 'v1'`,
-				"SELECT * FROM `resources` WHERE `group` = 'apps' AND `resource` = 'deployments' AND `version` = 'v1'",
+				`SELECT * FROM "apps_v1_deployments"`,
+				"SELECT * FROM `apps_v1_deployments`",
 				"",
 			},
 		},
@@ -307,8 +307,8 @@ func TestResourceStorage_deleteObject(t *testing.T) {
 			"",
 			"",
 			expected{
-				`DELETE FROM "resources" WHERE "cluster" = '' AND "group" = 'apps' AND "name" = '' AND "namespace" = '' AND "resource" = 'deployments' AND "version" = 'v1'`,
-				"DELETE FROM `resources` WHERE `cluster` = '' AND `group` = 'apps' AND `name` = '' AND `namespace` = '' AND `resource` = 'deployments' AND `version` = 'v1'",
+				`DELETE FROM "apps_v1_deployments" WHERE "cluster" = '' AND "name" = '' AND "namespace" = ''`,
+				"DELETE FROM `apps_v1_deployments` WHERE `cluster` = '' AND `name` = '' AND `namespace` = ''",
 				"",
 			},
 		},
@@ -319,8 +319,8 @@ func TestResourceStorage_deleteObject(t *testing.T) {
 			"ns-1",
 			"resource-1",
 			expected{
-				`DELETE FROM "resources" WHERE "cluster" = 'cluster-1' AND "group" = 'apps' AND "name" = 'resource-1' AND "namespace" = 'ns-1' AND "resource" = 'deployments' AND "version" = 'v1'`,
-				"DELETE FROM `resources` WHERE `cluster` = 'cluster-1' AND `group` = 'apps' AND `name` = 'resource-1' AND `namespace` = 'ns-1' AND `resource` = 'deployments' AND `version` = 'v1'",
+				`DELETE FROM "apps_v1_deployments" WHERE "cluster" = 'cluster-1' AND "name" = 'resource-1' AND "namespace" = 'ns-1'`,
+				"DELETE FROM `apps_v1_deployments` WHERE `cluster` = 'cluster-1' AND `name` = 'resource-1' AND `namespace` = 'ns-1'",
 				"",
 			},
 		},
diff --git a/pkg/storage/internalstorage/storage.go b/pkg/storage/internalstorage/storage.go
@@ -59,8 +59,8 @@ func (s *StorageFactory) NewResourceStorage(config *storage.ResourceStorageConfi
 		KeyFunc:    utils.GetKeyFunc(gvr, config.Namespaced),
 	}
 
-	// initEventCache is true when Apiserver starts, false when clustersynchro-manager starts
-	if initEventCache {
+	// SubscriberEnabled is true when Apiserver starts and middleware enabled
+	if middleware.SubscriberEnabled {
 		var cache *watchcomponents.EventCache
 		buffer := watchcomponents.GetMultiClusterEventPool().GetClusterBufferByGVR(gvr)
 		cachePool := watchcomponents.GetInitEventCachePool()
@@ -88,7 +88,7 @@ func (s *StorageFactory) NewResourceStorage(config *storage.ResourceStorageConfi
 
 		resourceStorage.buffer = buffer
 		resourceStorage.eventCache = cache
-	} else {
+	} else if middleware.PublisherEnabled { // PublisherEnabled is true when clustersynchro-manager starts and middleware enabled
 		err := middleware.GlobalPublisher.PublishTopic(gvr, config.Codec)
 		if err != nil {
 			return nil, err
diff --git a/pkg/synchromanager/clustersynchro/resource_synchro.go b/pkg/synchromanager/clustersynchro/resource_synchro.go
@@ -27,6 +27,7 @@ import (
 	"github.com/clusterpedia-io/clusterpedia/pkg/synchromanager/features"
 	"github.com/clusterpedia-io/clusterpedia/pkg/utils"
 	clusterpediafeature "github.com/clusterpedia-io/clusterpedia/pkg/utils/feature"
+	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/middleware"
 )
 
 type ResourceSynchro struct {
@@ -384,19 +385,23 @@ func (synchro *ResourceSynchro) handleResourceEvent(event *queue.Event) {
 				Published: true,
 			}
 			synchro.rvsLock.Unlock()
-			err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)
-			if err != nil {
-				return
+			if middleware.PublisherEnabled {
+				err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)
+				if err != nil {
+					return
+				}
 			}
 		}
 	} else {
 		handler, callback = synchro.deleteResource, func(_ runtime.Object, eventType watch.EventType) {
 			synchro.rvsLock.Lock()
 			delete(synchro.rvs, key)
 			synchro.rvsLock.Unlock()
-			err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)
-			if err != nil {
-				return
+			if middleware.PublisherEnabled {
+				err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)
+				if err != nil {
+					return
+				}
 			}
 		}
 	}
diff --git a/pkg/watcher/middleware/publisher.go b/pkg/watcher/middleware/publisher.go
@@ -11,6 +11,7 @@ import (
 )
 
 var GlobalPublisher Publisher
+var PublisherEnabled bool = false
 
 type Publisher interface {
 	InitPublisher(ctx context.Context) error
diff --git a/pkg/watcher/middleware/rabbitmq/register.go b/pkg/watcher/middleware/rabbitmq/register.go
@@ -15,7 +15,7 @@ const (
 	SubscribeerName = "rabbitmq"
 )
 
-func NewPulisher(mo *options.MiddlerwareOptions) (middleware.Publisher, error) {
+func NewPulisher(mo *options.MiddlewareOptions) (middleware.Publisher, error) {
 	if mo.MaxConnections <= 0 {
 		mo.MaxConnections = 3
 	}
@@ -34,7 +34,7 @@ func NewPulisher(mo *options.MiddlerwareOptions) (middleware.Publisher, error) {
 	return publisher, nil
 }
 
-func NewSubscriber(mo *options.MiddlerwareOptions) (middleware.Subscriber, error) {
+func NewSubscriber(mo *options.MiddlewareOptions) (middleware.Subscriber, error) {
 	if mo.MaxConnections <= 0 {
 		mo.MaxConnections = 3
 	}
diff --git a/pkg/watcher/middleware/subscriber.go b/pkg/watcher/middleware/subscriber.go
@@ -7,6 +7,7 @@ import (
 )
 
 var GlobalSubscriber Subscriber
+var SubscriberEnabled bool = false
 
 type Subscriber interface {
 	InitSubscriber(stopCh <-chan struct{}) error
diff --git a/pkg/watcher/options/options.go b/pkg/watcher/options/options.go
@@ -6,7 +6,8 @@ import (
 	"github.com/spf13/pflag"
 )
 
-type MiddlerwareOptions struct {
+type MiddlewareOptions struct {
+	Enabled                     bool // middleware enabled
 	Name                        string
 	ServerIp                    string
 	ServerPort                  int
@@ -20,11 +21,11 @@ type MiddlerwareOptions struct {
 	CacheSize                   int
 }
 
-func NewMiddlerwareOptions() *MiddlerwareOptions {
-	return &MiddlerwareOptions{Name: "apiserver", CacheSize: 100}
+func NewMiddlerwareOptions() *MiddlewareOptions {
+	return &MiddlewareOptions{Enabled: false, Name: "rabbitmq", CacheSize: 100}
 }
 
-func (o *MiddlerwareOptions) Validate() []error {
+func (o *MiddlewareOptions) Validate() []error {
 	if o == nil {
 		return nil
 	}
@@ -45,7 +46,8 @@ func (o *MiddlerwareOptions) Validate() []error {
 	return errors
 }
 
-func (o *MiddlerwareOptions) AddFlags(fs *pflag.FlagSet) {
+func (o *MiddlewareOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&o.Enabled, "middleware-enabled", o.Enabled, "middlerware enabled")
 	fs.StringVar(&o.Name, "middleware-name", o.Name, "middlerware name")
 	fs.StringVar(&o.ServerIp, "middleware-serverIp", o.ServerIp, "middlerware server Ip")
 	fs.IntVar(&o.ServerPort, "middleware-serverPort", o.ServerPort, "middlerware server port")
diff --git a/pkg/watcher/register.go b/pkg/watcher/register.go
@@ -8,8 +8,8 @@ import (
 	"github.com/clusterpedia-io/clusterpedia/pkg/watcher/options"
 )
 
-type NewPublisherFunc func(mo *options.MiddlerwareOptions) (middleware.Publisher, error)
-type NewSubscriberFunc func(mo *options.MiddlerwareOptions) (middleware.Subscriber, error)
+type NewPublisherFunc func(mo *options.MiddlewareOptions) (middleware.Publisher, error)
+type NewSubscriberFunc func(mo *options.MiddlewareOptions) (middleware.Subscriber, error)
 
 var publisherFuncs = make(map[string]NewPublisherFunc)
 var subscriberFuncs = make(map[string]NewSubscriberFunc)
@@ -33,7 +33,7 @@ func RegisterSubscriberFunc(name string, f NewSubscriberFunc) {
 	subscriberFuncs[name] = f
 }
 
-func NewPulisher(mo *options.MiddlerwareOptions) error {
+func NewPulisher(mo *options.MiddlewareOptions) error {
 	provider, ok := publisherFuncs[mo.Name]
 	if !ok {
 		return fmt.Errorf("publisher %s is unregistered", mo.Name)
@@ -48,7 +48,7 @@ func NewPulisher(mo *options.MiddlerwareOptions) error {
 	return nil
 }
 
-func NewSubscriber(mo *options.MiddlerwareOptions) error {
+func NewSubscriber(mo *options.MiddlewareOptions) error {
 	provider, ok := subscriberFuncs[mo.Name]
 	if !ok {
 		return fmt.Errorf("publisher %s is unregistered", mo.Name)
diff --git a/staging/src/github.com/clusterpedia-io/api/clusterpedia/v1beta1/types.go b/staging/src/github.com/clusterpedia-io/api/clusterpedia/v1beta1/types.go
@@ -23,6 +23,9 @@ type ListOptions struct {
 	// +optional
 	Namespaces string `json:"namespaces,omitempty"`
 
+	// +optional
+	ResourcePrefix string `json:"resourcePrefix,omitempty"`
+
 	// +optional
 	OrderBy string `json:"orderby,omitempty"`
 
diff --git a/staging/src/github.com/clusterpedia-io/api/clusterpedia/v1beta1/zz_generated.conversion.go b/staging/src/github.com/clusterpedia-io/api/clusterpedia/v1beta1/zz_generated.conversion.go

Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,7 @@ import (`
`27`	`27`	`"github.com/clusterpedia-io/clusterpedia/pkg/synchromanager/features"`
`28`	`28`	`"github.com/clusterpedia-io/clusterpedia/pkg/utils"`
`29`	`29`	`clusterpediafeature "github.com/clusterpedia-io/clusterpedia/pkg/utils/feature"`
	`30`	`+ "github.com/clusterpedia-io/clusterpedia/pkg/watcher/middleware"`
`30`	`31`	`)`
`31`	`32`
`32`	`33`	`type ResourceSynchro struct {`
`@@ -384,19 +385,23 @@ func (synchro ResourceSynchro) handleResourceEvent(event queue.Event) {`
`384`	`385`	`Published: true,`
`385`	`386`	`}`
`386`	`387`	`synchro.rvsLock.Unlock()`
`387`		`- err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)`
`388`		`- if err != nil {`
`389`		`- return`
	`388`	`+ if middleware.PublisherEnabled {`
	`389`	`+ err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)`
	`390`	`+ if err != nil {`
	`391`	`+ return`
	`392`	`+ }`
`390`	`393`	`}`
`391`	`394`	`}`
`392`	`395`	`} else {`
`393`	`396`	`handler, callback = synchro.deleteResource, func(_ runtime.Object, eventType watch.EventType) {`
`394`	`397`	`synchro.rvsLock.Lock()`
`395`	`398`	`delete(synchro.rvs, key)`
`396`	`399`	`synchro.rvsLock.Unlock()`
`397`		`- err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)`
`398`		`- if err != nil {`
`399`		`- return`
	`400`	`+ if middleware.PublisherEnabled {`
	`401`	`+ err := synchro.storage.ProcessEvent(context.TODO(), eventType, obj, synchro.cluster)`
	`402`	`+ if err != nil {`
	`403`	`+ return`
	`404`	`+ }`
`400`	`405`	`}`
`401`	`406`	`}`
`402`	`407`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,6 +11,7 @@ import (`
`11`	`11`	`)`
`12`	`12`
`13`	`13`	`var GlobalPublisher Publisher`
	`14`	`+var PublisherEnabled bool = false`
`14`	`15`
`15`	`16`	`type Publisher interface {`
`16`	`17`	`InitPublisher(ctx context.Context) error`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ const (`
`15`	`15`	`SubscribeerName = "rabbitmq"`
`16`	`16`	`)`
`17`	`17`
`18`		`-func NewPulisher(mo *options.MiddlerwareOptions) (middleware.Publisher, error) {`
	`18`	`+func NewPulisher(mo *options.MiddlewareOptions) (middleware.Publisher, error) {`
`19`	`19`	`if mo.MaxConnections <= 0 {`
`20`	`20`	`mo.MaxConnections = 3`
`21`	`21`	`}`
`@@ -34,7 +34,7 @@ func NewPulisher(mo *options.MiddlerwareOptions) (middleware.Publisher, error) {`
`34`	`34`	`return publisher, nil`
`35`	`35`	`}`
`36`	`36`
`37`		`-func NewSubscriber(mo *options.MiddlerwareOptions) (middleware.Subscriber, error) {`
	`37`	`+func NewSubscriber(mo *options.MiddlewareOptions) (middleware.Subscriber, error) {`
`38`	`38`	`if mo.MaxConnections <= 0 {`
`39`	`39`	`mo.MaxConnections = 3`
`40`	`40`	`}`