Merge pull request #4376 from wackxu/adjustconfig

auto adjust cloudcore config

4 files changed, 144 insertions, 59 deletions

diff --git a/cloud/cmd/cloudcore/app/server.go b/cloud/cmd/cloudcore/app/server.go
index 1efe20316..68da4ae06 100644
--- a/cloud/cmd/cloudcore/app/server.go
+++ b/cloud/cmd/cloudcore/app/server.go
@@ -32,6 +32,7 @@ import (
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/term"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/yaml"
 
 	"github.com/kubeedge/beehive/pkg/core"
 	beehiveContext "github.com/kubeedge/beehive/pkg/core/context"
@@ -106,6 +107,10 @@ kubernetes controller which manages devices so that the device metadata/status d
 
 			config.CommonConfig.TunnelPort = *tunnelport
 
+			if changed := v1alpha1.AdjustCloudCoreConfig(config); changed {
+				updateCloudCoreConfigMap(config)
+			}
+
 			gis := informers.GetInformersManager()
 
 			registerModules(config)
@@ -255,3 +260,26 @@ func negotiatePort(portRecord map[int]bool) int {
 		}
 	}
 }
+
+func updateCloudCoreConfigMap(c *v1alpha1.CloudCoreConfig) {
+	kubeClient := client.GetKubeClient()
+
+	cloudCoreCM, err := kubeClient.CoreV1().ConfigMaps(constants.SystemNamespace).Get(context.TODO(), constants.CloudConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		klog.Warningf("failed to get CloudCore configMap %s/%s", constants.SystemNamespace, constants.CloudConfigMapName)
+		return
+	}
+
+	configBytes, err := yaml.Marshal(c)
+	if err != nil {
+		klog.Errorf("Failed to marshal cloudcore config: %v", err)
+		return
+	}
+
+	cloudCoreCM.Data["cloudcore.yaml"] = string(configBytes)
+
+	_, err = kubeClient.CoreV1().ConfigMaps(constants.SystemNamespace).Update(context.TODO(), cloudCoreCM, metav1.UpdateOptions{})
+	if err != nil {
+		klog.Errorf("Failed to marshal cloudcore config: %v", err)
+	}
+}
diff --git a/common/constants/default.go b/common/constants/default.go
index 4a1b59b4b..cc94a3268 100644
--- a/common/constants/default.go
+++ b/common/constants/default.go
@@ -18,6 +18,8 @@ const (
 
 	SystemName      = "kubeedge"
 	SystemNamespace = SystemName
+
+	CloudConfigMapName = "cloudcore"
 )
 
 // Resources
@@ -84,26 +86,24 @@ const (
 	DefaultKubeNamespace           = v1.NamespaceAll
 	DefaultKubeQPS                 = 100.0
 	DefaultKubeBurst               = 200
+	DefaultNodeLimit               = 500
 	DefaultKubeUpdateNodeFrequency = 20
 
 	// EdgeController
 	DefaultUpdatePodStatusWorkers            = 1
 	DefaultUpdateNodeStatusWorkers           = 1
-	DefaultQueryConfigMapWorkers             = 4
-	DefaultQuerySecretWorkers                = 4
+	DefaultQueryConfigMapWorkers             = 100
+	DefaultQuerySecretWorkers                = 100
 	DefaultQueryPersistentVolumeWorkers      = 4
 	DefaultQueryPersistentVolumeClaimWorkers = 4
 	DefaultQueryVolumeAttachmentWorkers      = 4
-	DefaultCreateNodeWorkers                 = 4
-	DefaultPatchNodeWorkers                  = 4
-	DefaultQueryNodeWorkers                  = 4
+	DefaultCreateNodeWorkers                 = 100
 	DefaultUpdateNodeWorkers                 = 4
-	DefaultPatchPodWorkers                   = 4
-	DefaultDeletePodWorkers                  = 4
+	DefaultPatchPodWorkers                   = 100
+	DefaultDeletePodWorkers                  = 100
 	DefaultUpdateRuleStatusWorkers           = 4
-	DefaultCreateLeaseWorkers                = 4
-	DefaultQueryLeaseWorkers                 = 4
-	DefaultServiceAccountTokenWorkers        = 4
+	DefaultQueryLeaseWorkers                 = 100
+	DefaultServiceAccountTokenWorkers        = 100
 
 	DefaultUpdatePodStatusBuffer            = 1024
 	DefaultUpdateNodeStatusBuffer           = 1024
@@ -113,12 +113,9 @@ const (
 	DefaultQueryPersistentVolumeClaimBuffer = 1024
 	DefaultQueryVolumeAttachmentBuffer      = 1024
 	DefaultCreateNodeBuffer                 = 1024
-	DefaultPatchNodeBuffer                  = 1024
-	DefaultQueryNodeBuffer                  = 1024
 	DefaultUpdateNodeBuffer                 = 1024
 	DefaultPatchPodBuffer                   = 1024
 	DefaultDeletePodBuffer                  = 1024
-	DefaultCreateLeaseBuffer                = 1024
 	DefaultQueryLeaseBuffer                 = 1024
 	DefaultServiceAccountTokenBuffer        = 1024
 
diff --git a/pkg/apis/componentconfig/cloudcore/v1alpha1/default.go b/pkg/apis/componentconfig/cloudcore/v1alpha1/default.go
index fae69f788..d18a38ad2 100644
--- a/pkg/apis/componentconfig/cloudcore/v1alpha1/default.go
+++ b/pkg/apis/componentconfig/cloudcore/v1alpha1/default.go
@@ -43,14 +43,14 @@ func NewDefaultCloudCoreConfig() *CloudCoreConfig {
 		},
 		KubeAPIConfig: &KubeAPIConfig{
 			ContentType: constants.DefaultKubeContentType,
-			QPS:         constants.DefaultKubeQPS,
-			Burst:       constants.DefaultKubeBurst,
+			QPS:         5 * constants.DefaultNodeLimit,
+			Burst:       10 * constants.DefaultNodeLimit,
 		},
 		Modules: &Modules{
 			CloudHub: &CloudHub{
 				Enable:                  true,
 				KeepaliveInterval:       30,
-				NodeLimit:               1000,
+				NodeLimit:               constants.DefaultNodeLimit,
 				TLSCAFile:               constants.DefaultCAFile,
 				TLSCAKeyFile:            constants.DefaultCAKeyFile,
 				TLSCertFile:             constants.DefaultCertFile,
@@ -84,48 +84,8 @@ func NewDefaultCloudCoreConfig() *CloudCoreConfig {
 			EdgeController: &EdgeController{
 				Enable:              true,
 				NodeUpdateFrequency: 10,
-				Buffer: &EdgeControllerBuffer{
-					UpdatePodStatus:            constants.DefaultUpdatePodStatusBuffer,
-					UpdateNodeStatus:           constants.DefaultUpdateNodeStatusBuffer,
-					QueryConfigMap:             constants.DefaultQueryConfigMapBuffer,
-					QuerySecret:                constants.DefaultQuerySecretBuffer,
-					PodEvent:                   constants.DefaultPodEventBuffer,
-					ConfigMapEvent:             constants.DefaultConfigMapEventBuffer,
-					SecretEvent:                constants.DefaultSecretEventBuffer,
-					RulesEvent:                 constants.DefaultRulesEventBuffer,
-					RuleEndpointsEvent:         constants.DefaultRuleEndpointsEventBuffer,
-					QueryPersistentVolume:      constants.DefaultQueryPersistentVolumeBuffer,
-					QueryPersistentVolumeClaim: constants.DefaultQueryPersistentVolumeClaimBuffer,
-					QueryVolumeAttachment:      constants.DefaultQueryVolumeAttachmentBuffer,
-					CreateNode:                 constants.DefaultCreateNodeBuffer,
-					PatchNode:                  constants.DefaultPatchNodeBuffer,
-					QueryNode:                  constants.DefaultQueryNodeBuffer,
-					UpdateNode:                 constants.DefaultUpdateNodeBuffer,
-					PatchPod:                   constants.DefaultPatchPodBuffer,
-					DeletePod:                  constants.DefaultDeletePodBuffer,
-					CreateLease:                constants.DefaultCreateLeaseBuffer,
-					QueryLease:                 constants.DefaultQueryLeaseBuffer,
-					ServiceAccountToken:        constants.DefaultServiceAccountTokenBuffer,
-				},
-				Load: &EdgeControllerLoad{
-					UpdatePodStatusWorkers:            constants.DefaultUpdatePodStatusWorkers,
-					UpdateNodeStatusWorkers:           constants.DefaultUpdateNodeStatusWorkers,
-					QueryConfigMapWorkers:             constants.DefaultQueryConfigMapWorkers,
-					QuerySecretWorkers:                constants.DefaultQuerySecretWorkers,
-					QueryPersistentVolumeWorkers:      constants.DefaultQueryPersistentVolumeWorkers,
-					QueryPersistentVolumeClaimWorkers: constants.DefaultQueryPersistentVolumeClaimWorkers,
-					QueryVolumeAttachmentWorkers:      constants.DefaultQueryVolumeAttachmentWorkers,
-					QueryNodeWorkers:                  constants.DefaultQueryNodeWorkers,
-					CreateNodeWorkers:                 constants.DefaultCreateNodeWorkers,
-					PatchNodeWorkers:                  constants.DefaultPatchNodeWorkers,
-					UpdateNodeWorkers:                 constants.DefaultUpdateNodeWorkers,
-					PatchPodWorkers:                   constants.DefaultPatchPodWorkers,
-					DeletePodWorkers:                  constants.DefaultDeletePodWorkers,
-					CreateLeaseWorkers:                constants.DefaultCreateLeaseWorkers,
-					QueryLeaseWorkers:                 constants.DefaultQueryLeaseWorkers,
-					UpdateRuleStatusWorkers:           constants.DefaultUpdateRuleStatusWorkers,
-					ServiceAccountTokenWorkers:        constants.DefaultServiceAccountTokenWorkers,
-				},
+				Buffer:              getDefaultEdgeControllerBuffer(constants.DefaultNodeLimit),
+				Load:                getDefaultEdgeControllerLoad(constants.DefaultNodeLimit),
 			},
 			DeviceController: &DeviceController{
 				Enable: true,
@@ -180,6 +140,106 @@ func NewDefaultCloudCoreConfig() *CloudCoreConfig {
 	return c
 }
 
+// NodeLimit is a maximum number of edge node that can connect to the single CloudCore
+// instance. You should take this parameter seriously, because this parameter is closely
+// related to the number of goroutines for upstream message processing.
+// getDefaultEdgeControllerLoad return Default EdgeControllerLoad based on nodeLimit
+func getDefaultEdgeControllerLoad(nodeLimit int32) *EdgeControllerLoad {
+	return &EdgeControllerLoad{
+		UpdatePodStatusWorkers:            constants.DefaultUpdatePodStatusWorkers,
+		UpdateNodeStatusWorkers:           constants.DefaultUpdateNodeStatusWorkers,
+		QueryConfigMapWorkers:             constants.DefaultQueryConfigMapWorkers,
+		QuerySecretWorkers:                constants.DefaultQuerySecretWorkers,
+		QueryPersistentVolumeWorkers:      constants.DefaultQueryPersistentVolumeWorkers,
+		QueryPersistentVolumeClaimWorkers: constants.DefaultQueryPersistentVolumeClaimWorkers,
+		QueryVolumeAttachmentWorkers:      constants.DefaultQueryVolumeAttachmentWorkers,
+		QueryNodeWorkers:                  nodeLimit,
+		CreateNodeWorkers:                 constants.DefaultCreateNodeWorkers,
+		PatchNodeWorkers:                  100 + nodeLimit/50,
+		UpdateNodeWorkers:                 constants.DefaultUpdateNodeWorkers,
+		PatchPodWorkers:                   constants.DefaultPatchPodWorkers,
+		DeletePodWorkers:                  constants.DefaultDeletePodWorkers,
+		CreateLeaseWorkers:                nodeLimit,
+		QueryLeaseWorkers:                 constants.DefaultQueryLeaseWorkers,
+		UpdateRuleStatusWorkers:           constants.DefaultUpdateRuleStatusWorkers,
+		ServiceAccountTokenWorkers:        constants.DefaultServiceAccountTokenWorkers,
+	}
+}
+
+// getDefaultEdgeControllerBuffer return Default EdgeControllerBuffer based on nodeLimit
+func getDefaultEdgeControllerBuffer(nodeLimit int32) *EdgeControllerBuffer {
+	return &EdgeControllerBuffer{
+		UpdatePodStatus:            constants.DefaultUpdatePodStatusBuffer,
+		UpdateNodeStatus:           constants.DefaultUpdateNodeStatusBuffer,
+		QueryConfigMap:             constants.DefaultQueryConfigMapBuffer,
+		QuerySecret:                constants.DefaultQuerySecretBuffer,
+		PodEvent:                   constants.DefaultPodEventBuffer,
+		ConfigMapEvent:             constants.DefaultConfigMapEventBuffer,
+		SecretEvent:                constants.DefaultSecretEventBuffer,
+		RulesEvent:                 constants.DefaultRulesEventBuffer,
+		RuleEndpointsEvent:         constants.DefaultRuleEndpointsEventBuffer,
+		QueryPersistentVolume:      constants.DefaultQueryPersistentVolumeBuffer,
+		QueryPersistentVolumeClaim: constants.DefaultQueryPersistentVolumeClaimBuffer,
+		QueryVolumeAttachment:      constants.DefaultQueryVolumeAttachmentBuffer,
+		CreateNode:                 constants.DefaultCreateNodeBuffer,
+		PatchNode:                  1024 + nodeLimit/2,
+		QueryNode:                  1024 + nodeLimit,
+		UpdateNode:                 constants.DefaultUpdateNodeBuffer,
+		PatchPod:                   constants.DefaultPatchPodBuffer,
+		DeletePod:                  constants.DefaultDeletePodBuffer,
+		CreateLease:                1024 + nodeLimit,
+		QueryLease:                 constants.DefaultQueryLeaseBuffer,
+		ServiceAccountToken:        constants.DefaultServiceAccountTokenBuffer,
+	}
+}
+
+func AdjustCloudCoreConfig(c *CloudCoreConfig) bool {
+	changed := false
+	nodeLimit := c.Modules.CloudHub.NodeLimit
+
+	if c.KubeAPIConfig.QPS != 5*nodeLimit {
+		changed = true
+		c.KubeAPIConfig.QPS = 5 * nodeLimit
+	}
+
+	if c.KubeAPIConfig.Burst != 10*nodeLimit {
+		changed = true
+		c.KubeAPIConfig.Burst = 10 * nodeLimit
+	}
+
+	if c.Modules.EdgeController.Load.QueryNodeWorkers < nodeLimit {
+		changed = true
+		c.Modules.EdgeController.Load.QueryNodeWorkers = nodeLimit
+	}
+
+	if c.Modules.EdgeController.Load.PatchNodeWorkers < 100+nodeLimit/50 {
+		changed = true
+		c.Modules.EdgeController.Load.PatchNodeWorkers = 100 + nodeLimit/50
+	}
+
+	if c.Modules.EdgeController.Load.CreateLeaseWorkers < nodeLimit {
+		changed = true
+		c.Modules.EdgeController.Load.CreateLeaseWorkers = nodeLimit
+	}
+
+	if c.Modules.EdgeController.Buffer.PatchNode < 1024+nodeLimit/2 {
+		changed = true
+		c.Modules.EdgeController.Buffer.PatchNode = 1024 + nodeLimit/2
+	}
+
+	if c.Modules.EdgeController.Buffer.QueryNode < 1024+nodeLimit {
+		changed = true
+		c.Modules.EdgeController.Buffer.QueryNode = 1024 + nodeLimit
+	}
+
+	if c.Modules.EdgeController.Buffer.CreateLease < 1024+nodeLimit {
+		changed = true
+		c.Modules.EdgeController.Buffer.CreateLease = 1024 + nodeLimit
+	}
+
+	return changed
+}
+
 // NewMinCloudCoreConfig returns a min CloudCoreConfig object
 func NewMinCloudCoreConfig() *CloudCoreConfig {
 	advertiseAddress, _ := utilnet.ChooseHostInterface()
diff --git a/pkg/apis/componentconfig/cloudcore/v1alpha1/types.go b/pkg/apis/componentconfig/cloudcore/v1alpha1/types.go
index a0c2bfc8c..738008d0c 100644
--- a/pkg/apis/componentconfig/cloudcore/v1alpha1/types.go
+++ b/pkg/apis/componentconfig/cloudcore/v1alpha1/types.go
@@ -113,7 +113,7 @@ type CloudHub struct {
 	// KeepaliveInterval indicates keep-alive interval (second)
 	// default 30
 	KeepaliveInterval int32 `json:"keepaliveInterval,omitempty"`
-	// NodeLimit indicates node limit
+	// NodeLimit is a maximum number of edge node that can connect to the single CloudCore
 	// default 1000
 	NodeLimit int32 `json:"nodeLimit,omitempty"`
 	// TLSCAFile indicates ca file path